From patchwork Tue Jul 21 22:01:24 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Kieran Bingham X-Patchwork-Id: 8913 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 4CB11C2E68 for ; Tue, 21 Jul 2020 22:01:38 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 06613609A9; Wed, 22 Jul 2020 00:01:38 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="WWbxNmv2"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id EB4DB60943 for ; Wed, 22 Jul 2020 00:01:32 +0200 (CEST) Received: from localhost.localdomain (cpc89242-aztw30-2-0-cust488.18-1.cable.virginm.net [86.31.129.233]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 2049D51A; Wed, 22 Jul 2020 00:01:32 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1595368892; bh=8FCCc/j7kEX8ddGZY6kMswMJEatvfKkxggXUodGUX+o=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=WWbxNmv223Tx06jMQsqLvhbhXbRIvzRGYwtSvZpLT8eFTPIhkvDxw1Iw6qac9toaS nRJ9tzt0siSW9QP2aatxxYYae/mAwhtdCYk7N9JkjVLBJDef8Mo6jxJDw8JJ0YD9Up attsMZyLPRwvnKktVwxGASO1Rt0AXAxV66lE5FRE= From: Kieran Bingham To: libcamera devel Date: Tue, 21 Jul 2020 23:01:24 +0100 Message-Id: <20200721220126.202065-5-kieran.bingham@ideasonboard.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200721220126.202065-1-kieran.bingham@ideasonboard.com> References: <20200721220126.202065-1-kieran.bingham@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH 4/6] android: camera_device: Support MJPEG stream construction X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" MJPEG streams must be created referencing a libcamera stream. This stream may already be provided by the request configuration, in which case the existing stream is utilised. If no compatible stream is available to encode, a new stream is requested from the libcamera configuration. Signed-off-by: Kieran Bingham --- src/android/camera_device.cpp | 158 +++++++++++++++++++++++++++++++++- src/android/camera_device.h | 8 ++ 2 files changed, 162 insertions(+), 4 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 56652ac57676..7323d4e58f68 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,7 @@ #include "camera_device.h" #include "camera_ops.h" +#include #include #include @@ -21,6 +22,8 @@ #include "camera_metadata.h" #include "system/graphics.h" +#include "jpeg/compressor_jpeg.h" + using namespace libcamera; namespace { @@ -1004,6 +1007,7 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) */ unsigned int streamIndex = 0; + /* First handle all non-MJPEG streams */ for (unsigned int i = 0; i < stream_list->num_streams; ++i) { camera3_stream_t *stream = stream_list->streams[i]; @@ -1019,6 +1023,18 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) if (!format.isValid()) return -EINVAL; + stream->priv = static_cast(&streams_[i]); + streams_[i].format = format; + streams_[i].size = Size(stream->width, stream->height); + + /* Defer handling of MJPEG streams until all others are known. */ + if (format == formats::MJPEG) { + LOG(HAL, Info) << "Handling MJPEG stream"; + + streams_[i].index = -1; + continue; + } + StreamConfiguration streamConfiguration; streamConfiguration.size.width = stream->width; @@ -1028,7 +1044,61 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) config_->addConfiguration(streamConfiguration); streams_[i].index = streamIndex++; - stream->priv = static_cast(&streams_[i]); + } + + /* Now handle MJPEG streams, adding a new stream if required. */ + for (unsigned int i = 0; i < stream_list->num_streams; ++i) { + camera3_stream_t *stream = stream_list->streams[i]; + CameraStream *cameraStream = &streams_[i]; + + /* Only process MJPEG streams */ + if (cameraStream->format != formats::MJPEG) + continue; + + bool match = false; + + /* Search for a compatible stream */ + for (unsigned int j = 0; j < config_->size(); j++) { + StreamConfiguration &cfg = config_->at(j); + + /* + * The PixelFormat must also be compatible with the + * encoder. + */ + if (cfg.size == cameraStream->size) { + LOG(HAL, Info) + << "Stream " << i + << " using libcamera stream " + << j; + + match = true; + cameraStream->index = j; + } + } + + /* + * Without a compatible match for JPEG encoding we must + * introduce a new stream to satisfy the request requirements. + */ + if (!match) { + StreamConfiguration streamConfiguration; + + /* + * \todo: The pixelFormat should be a 'best-fit' choice + * and may require a validation cycle. This is not yet + * handled, and should be considered as part of any + * stream configuration reworks. + */ + streamConfiguration.size.width = stream->width; + streamConfiguration.size.height = stream->height; + streamConfiguration.pixelFormat = formats::NV12; + + LOG(HAL, Info) << "Adding " << streamConfiguration.toString() + << " for MJPEG support"; + + config_->addConfiguration(streamConfiguration); + streams_[i].index = streamIndex++; + } } switch (config_->validate()) { @@ -1059,6 +1129,18 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) /* Use the bufferCount confirmed by the validation process. */ stream->max_buffers = cfg.bufferCount; + + /* + * Construct a software compressor for MJPEG streams from the + * chosen libcamera source stream. + */ + if (cameraStream->format == formats::MJPEG) { + cameraStream->jpeg = new CompressorJPEG(); + cameraStream->jpeg->configure(cfg); + } else { + /* Either construct this correctly, or use a better interface */ + cameraStream->jpeg = nullptr; + } } /* @@ -1112,6 +1194,9 @@ FrameBuffer *CameraDevice::createFrameBuffer(const buffer_handle_t camera3buffer int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Request) { + LOG(HAL, Error) << "Received request " << camera3Request->frame_number + << " with " << camera3Request->num_output_buffers << " buffers"; + if (!camera3Request->num_output_buffers) { LOG(HAL, Error) << "No output buffers provided"; return -EINVAL; @@ -1158,6 +1243,10 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques descriptor->buffers[i].stream = camera3Buffers[i].stream; descriptor->buffers[i].buffer = camera3Buffers[i].buffer; + /* Software streams are handled after hardware streams complete. */ + if (cameraStream->format == formats::MJPEG) + continue; + /* * Create a libcamera buffer using the dmabuf descriptors of * the camera3Buffer for each stream. The FrameBuffer is @@ -1190,11 +1279,40 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques return 0; } +static CompressedImage mapAndroidBlobBuffer(const buffer_handle_t camera3buffer) +{ + CompressedImage img; + + /* ANDROID_JPEG_MAX_SIZE */ + unsigned int length = int32_t{13 << 20}; + + /* Take only the first plane */ + void *memory = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, + camera3buffer->data[0], 0); + + img.length = length; + img.data = static_cast(memory); + + return img; +} + +static void unmapAndroidBlobBuffer(CompressedImage *img) +{ + munmap(img->data, img->length); + img->data = nullptr; + img->length = 0; +} + void CameraDevice::requestComplete(Request *request) { const std::map &buffers = request->buffers(); camera3_buffer_status status = CAMERA3_BUFFER_STATUS_OK; std::unique_ptr resultMetadata; + Camera3RequestDescriptor *descriptor = + reinterpret_cast(request->cookie()); + + + LOG(HAL, Error) << "Request completed:..."; if (request->status() != Request::RequestComplete) { LOG(HAL, Error) << "Request not succesfully completed: " @@ -1202,10 +1320,41 @@ void CameraDevice::requestComplete(Request *request) status = CAMERA3_BUFFER_STATUS_ERROR; } - /* Prepare to call back the Android camera stack. */ - Camera3RequestDescriptor *descriptor = - reinterpret_cast(request->cookie()); + /* Handle any JPEG compression */ + for (unsigned int i = 0; i < descriptor->numBuffers; ++i) { + CameraStream *cameraStream = + static_cast(descriptor->buffers[i].stream->priv); + Compressor *compressor = cameraStream->jpeg; + /* Only handle streams with compression enabled. */ + if (!compressor) + continue; + + /* \todo: Optimise this dance routine, just to get the stream/buffer ... */ + StreamConfiguration *streamConfiguration = &config_->at(cameraStream->index); + Stream *stream = streamConfiguration->stream(); + FrameBuffer *buffer = request->findBuffer(stream); + if (!buffer) { + LOG(HAL, Error) << "Failed to find a source stream buffer"; + continue; + } + + CompressedImage output = mapAndroidBlobBuffer(*descriptor->buffers[i].buffer); + if (output.data == MAP_FAILED) { + LOG(HAL, Error) << "Failed to mmap android blob buffer of length " << output.length; + continue; + } + + int ret = compressor->compress(buffer, &output); + if (ret) { + LOG(HAL, Error) << "Failed to compress stream image"; + status = CAMERA3_BUFFER_STATUS_ERROR; + } + + unmapAndroidBlobBuffer(&output); + } + + /* Prepare to call back the Android camera stack. */ camera3_capture_result_t captureResult = {}; captureResult.frame_number = descriptor->frameNumber; captureResult.num_output_buffers = descriptor->numBuffers; @@ -1246,6 +1395,7 @@ void CameraDevice::requestComplete(Request *request) descriptor->buffers[0].stream); } + callbacks_->process_capture_result(callbacks_, &captureResult); delete descriptor; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index 5b8b9c3e26e2..1973adaa2b4b 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -23,6 +23,8 @@ #include "libcamera/internal/log.h" #include "libcamera/internal/message.h" +#include "jpeg/compressor_jpeg.h" + class CameraMetadata; struct CameraStream { @@ -32,6 +34,12 @@ struct CameraStream { * one or more streams to the Android framework. */ unsigned int index; + + libcamera::PixelFormat format; + libcamera::Size size; + + /* Make sure this gets destructed correctly */ + CompressorJPEG *jpeg; }; class CameraDevice : protected libcamera::Loggable