Patch Detail
Show a patch.
GET /api/1.1/patches/9262/?format=api
{ "id": 9262, "url": "https://patchwork.libcamera.org/api/1.1/patches/9262/?format=api", "web_url": "https://patchwork.libcamera.org/patch/9262/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20200806145019.290878-3-kieran.bingham@ideasonboard.com>", "date": "2020-08-06T14:50:19", "name": "[libcamera-devel,v5,2/2] android: camera_device: Support MJPEG stream construction", "commit_ref": "83ae84eb13a2e057fd28f14300b57ff99544084c", "pull_url": null, "state": "accepted", "archived": false, "hash": "7ff2646786e721f05d601f1065325789f243be81", "submitter": { "id": 4, "url": "https://patchwork.libcamera.org/api/1.1/people/4/?format=api", "name": "Kieran Bingham", "email": "kieran.bingham@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/9262/mbox/", "series": [ { "id": 1207, "url": "https://patchwork.libcamera.org/api/1.1/series/1207/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1207", "date": "2020-08-06T14:50:17", "name": "android: JPEG suppert (Encoder)", "version": 5, "mbox": "https://patchwork.libcamera.org/series/1207/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/9262/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/9262/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 57A9BBD87B\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 6 Aug 2020 14:50:27 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7254C60E00;\n\tThu, 6 Aug 2020 16:50:26 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 40E4D60495\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 6 Aug 2020 16:50:24 +0200 (CEST)", "from Q.local (cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net\n\t[86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C877BAC8;\n\tThu, 6 Aug 2020 16:50:23 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"LE47bPDL\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1596725424;\n\tbh=kVNZom8kenDkpWcC43tpJPOmb/RVwkp0GiwaZk0GNaY=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=LE47bPDLrwYFmTIiwf69+VTV+/+DyX4zFMMeKx/mTnHl1mjHd1rl1k4YgOCtHEVnk\n\tg1TojxdkE5LauYiqy9B6wq91qTmbJprHQirtmdKMEviW2zUDgq2jlOlgM1KIInISiq\n\tDNeAsKK30UFI+6n7G7a7V2FV0XRcMCgDBFjT0AWE=", "From": "Kieran Bingham <kieran.bingham@ideasonboard.com>", "To": "libcamera devel <libcamera-devel@lists.libcamera.org>", "Date": "Thu, 6 Aug 2020 15:50:19 +0100", "Message-Id": "<20200806145019.290878-3-kieran.bingham@ideasonboard.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20200806145019.290878-1-kieran.bingham@ideasonboard.com>", "References": "<20200805151507.227503-1-kieran.bingham@ideasonboard.com>\n\t<20200806145019.290878-1-kieran.bingham@ideasonboard.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v5 2/2] android: camera_device: Support\n\tMJPEG stream construction", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "MJPEG streams must be created referencing a libcamera stream.\nThis stream may already be provided by the request configuration,\nin which case the existing stream is utilised.\n\nIf no compatible stream is available to encode, a new stream is requested\nfrom the libcamera configuration.\n\nReviewed-by: Jacopo Mondi <jacopo@jmondi.org>\nSigned-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\n---\n src/android/camera_device.cpp | 230 ++++++++++++++++++++++++++++++++--\n src/android/camera_device.h | 12 ++\n 2 files changed, 233 insertions(+), 9 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex e78fecf46f8d..d9183501228f 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -8,6 +8,7 @@\n #include \"camera_device.h\"\n #include \"camera_ops.h\"\n \n+#include <sys/mman.h>\n #include <tuple>\n #include <vector>\n \n@@ -22,6 +23,8 @@\n #include \"camera_metadata.h\"\n #include \"system/graphics.h\"\n \n+#include \"jpeg/encoder_libjpeg.h\"\n+\n using namespace libcamera;\n \n namespace {\n@@ -129,6 +132,52 @@ const std::map<int, const Camera3Format> camera3FormatsMap = {\n \n LOG_DECLARE_CATEGORY(HAL);\n \n+class MappedCamera3Buffer : public MappedBuffer\n+{\n+public:\n+\tMappedCamera3Buffer(const buffer_handle_t camera3buffer, int flags);\n+};\n+\n+MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer,\n+\t\t\t\t\t int flags)\n+{\n+\tmaps_.reserve(camera3buffer->numFds);\n+\terror_ = 0;\n+\n+\tfor (int i = 0; i < camera3buffer->numFds; i++) {\n+\t\tif (camera3buffer->data[i] == -1)\n+\t\t\tcontinue;\n+\n+\t\toff_t length = lseek(camera3buffer->data[i], 0, SEEK_END);\n+\t\tif (length < 0) {\n+\t\t\terror_ = -errno;\n+\t\t\tLOG(HAL, Error) << \"Failed to query plane length\";\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tvoid *address = mmap(nullptr, length, flags, MAP_SHARED,\n+\t\t\t\t camera3buffer->data[i], 0);\n+\t\tif (address == MAP_FAILED) {\n+\t\t\terror_ = -errno;\n+\t\t\tLOG(HAL, Error) << \"Failed to mmap plane\";\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tmaps_.emplace_back(static_cast<uint8_t *>(address),\n+\t\t\t\t static_cast<size_t>(length));\n+\t}\n+}\n+\n+CameraStream::CameraStream(PixelFormat f, Size s)\n+\t: index(-1), format(f), size(s), jpeg(nullptr)\n+{\n+}\n+\n+CameraStream::~CameraStream()\n+{\n+\tdelete jpeg;\n+};\n+\n /*\n * \\struct Camera3RequestDescriptor\n *\n@@ -167,6 +216,12 @@ CameraDevice::CameraDevice(unsigned int id, const std::shared_ptr<Camera> &camer\n \t facing_(CAMERA_FACING_FRONT), orientation_(0)\n {\n \tcamera_->requestCompleted.connect(this, &CameraDevice::requestComplete);\n+\n+\t/*\n+\t * \\todo Determine a more accurate value for this during\n+\t * streamConfiguration.\n+\t */\n+\tmaxJpegBufferSize_ = 13 << 20; /* 13631488 from USB HAL */\n }\n \n CameraDevice::~CameraDevice()\n@@ -417,10 +472,10 @@ std::tuple<uint32_t, uint32_t> CameraDevice::calculateStaticMetadataSize()\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 50 entries, 671 bytes of static metadata\n+\t * Currently: 51 entries, 687 bytes of static metadata\n \t */\n-\tuint32_t numEntries = 50;\n-\tuint32_t byteSize = 671;\n+\tuint32_t numEntries = 51;\n+\tuint32_t byteSize = 687;\n \n \t/*\n \t * Calculate space occupation in bytes for dynamically built metadata\n@@ -576,6 +631,12 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\t\t\t availableThumbnailSizes.data(),\n \t\t\t\t availableThumbnailSizes.size());\n \n+\t/*\n+\t * \\todo Calculate the maximum JPEG buffer size by asking the encoder\n+\t * giving the maximum frame size required.\n+\t */\n+\tstaticMetadata_->addEntry(ANDROID_JPEG_MAX_SIZE, &maxJpegBufferSize_, 1);\n+\n \t/* Sensor static metadata. */\n \tint32_t pixelArraySize[] = {\n \t\t2592, 1944,\n@@ -789,6 +850,7 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_CONTROL_AWB_LOCK_AVAILABLE,\n \t\tANDROID_CONTROL_AVAILABLE_MODES,\n \t\tANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES,\n+\t\tANDROID_JPEG_MAX_SIZE,\n \t\tANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_SENSITIVITY_RANGE,\n@@ -860,6 +922,9 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_SENSOR_EXPOSURE_TIME,\n \t\tANDROID_STATISTICS_LENS_SHADING_MAP_MODE,\n \t\tANDROID_STATISTICS_SCENE_FLICKER,\n+\t\tANDROID_JPEG_SIZE,\n+\t\tANDROID_JPEG_QUALITY,\n+\t\tANDROID_JPEG_ORIENTATION,\n \t};\n \tstaticMetadata_->addEntry(ANDROID_REQUEST_AVAILABLE_RESULT_KEYS,\n \t\t\t\t availableResultKeys.data(),\n@@ -1052,8 +1117,10 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t */\n \tunsigned int streamIndex = 0;\n \n+\t/* First handle all non-MJPEG streams. */\n \tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n \t\tcamera3_stream_t *stream = stream_list->streams[i];\n+\t\tSize size(stream->width, stream->height);\n \n \t\tPixelFormat format = toPixelFormat(stream->format);\n \n@@ -1067,16 +1134,71 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t\tif (!format.isValid())\n \t\t\treturn -EINVAL;\n \n+\t\tstreams_.emplace_back(format, size);\n+\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\n+\t\t/* Defer handling of MJPEG streams until all others are known. */\n+\t\tif (format == formats::MJPEG)\n+\t\t\tcontinue;\n+\n \t\tStreamConfiguration streamConfiguration;\n \n-\t\tstreamConfiguration.size.width = stream->width;\n-\t\tstreamConfiguration.size.height = stream->height;\n+\t\tstreamConfiguration.size = size;\n \t\tstreamConfiguration.pixelFormat = format;\n \n \t\tconfig_->addConfiguration(streamConfiguration);\n \n \t\tstreams_[i].index = streamIndex++;\n-\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\t}\n+\n+\t/* Now handle MJPEG streams, adding a new stream if required. */\n+\tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n+\t\tcamera3_stream_t *stream = stream_list->streams[i];\n+\t\tbool match = false;\n+\n+\t\tif (streams_[i].format != formats::MJPEG)\n+\t\t\tcontinue;\n+\n+\t\t/* Search for a compatible stream */\n+\t\tfor (unsigned int j = 0; j < config_->size(); j++) {\n+\t\t\tStreamConfiguration &cfg = config_->at(j);\n+\n+\t\t\t/*\n+\t\t\t * \\todo The PixelFormat must also be compatible with\n+\t\t\t * the encoder.\n+\t\t\t */\n+\t\t\tif (cfg.size == streams_[i].size) {\n+\t\t\t\tLOG(HAL, Info) << \"Stream \" << i\n+\t\t\t\t\t << \" using libcamera stream \" << j;\n+\n+\t\t\t\tmatch = true;\n+\t\t\t\tstreams_[i].index = j;\n+\t\t\t}\n+\t\t}\n+\n+\t\t/*\n+\t\t * Without a compatible match for JPEG encoding we must\n+\t\t * introduce a new stream to satisfy the request requirements.\n+\t\t */\n+\t\tif (!match) {\n+\t\t\tStreamConfiguration streamConfiguration;\n+\n+\t\t\t/*\n+\t\t\t * \\todo The pixelFormat should be a 'best-fit' choice\n+\t\t\t * and may require a validation cycle. This is not yet\n+\t\t\t * handled, and should be considered as part of any\n+\t\t\t * stream configuration reworks.\n+\t\t\t */\n+\t\t\tstreamConfiguration.size.width = stream->width;\n+\t\t\tstreamConfiguration.size.height = stream->height;\n+\t\t\tstreamConfiguration.pixelFormat = formats::NV12;\n+\n+\t\t\tLOG(HAL, Info) << \"Adding \" << streamConfiguration.toString()\n+\t\t\t\t << \" for MJPEG support\";\n+\n+\t\t\tconfig_->addConfiguration(streamConfiguration);\n+\t\t\tstreams_[i].index = streamIndex++;\n+\t\t}\n \t}\n \n \tswitch (config_->validate()) {\n@@ -1103,6 +1225,20 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \n \t\t/* Use the bufferCount confirmed by the validation process. */\n \t\tstream->max_buffers = cfg.bufferCount;\n+\n+\t\t/*\n+\t\t * Construct a software encoder for MJPEG streams from the\n+\t\t * chosen libcamera source stream.\n+\t\t */\n+\t\tif (cameraStream->format == formats::MJPEG) {\n+\t\t\tcameraStream->jpeg = new EncoderLibJpeg();\n+\t\t\tint ret = cameraStream->jpeg->configure(cfg);\n+\t\t\tif (ret) {\n+\t\t\t\tLOG(HAL, Error)\n+\t\t\t\t\t<< \"Failed to configure encoder\";\n+\t\t\t\treturn ret;\n+\t\t\t}\n+\t\t}\n \t}\n \n \t/*\n@@ -1197,6 +1333,10 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tdescriptor->buffers[i].stream = camera3Buffers[i].stream;\n \t\tdescriptor->buffers[i].buffer = camera3Buffers[i].buffer;\n \n+\t\t/* Software streams are handled after hardware streams complete. */\n+\t\tif (cameraStream->format == formats::MJPEG)\n+\t\t\tcontinue;\n+\n \t\t/*\n \t\t * Create a libcamera buffer using the dmabuf descriptors of\n \t\t * the camera3Buffer for each stream. The FrameBuffer is\n@@ -1253,8 +1393,80 @@ void CameraDevice::requestComplete(Request *request)\n \tresultMetadata = getResultMetadata(descriptor->frameNumber,\n \t\t\t\t\t buffer->metadata().timestamp);\n \n-\t/* Prepare to call back the Android camera stack. */\n+\t/* Handle any JPEG compression. */\n+\tfor (unsigned int i = 0; i < descriptor->numBuffers; ++i) {\n+\t\tCameraStream *cameraStream =\n+\t\t\tstatic_cast<CameraStream *>(descriptor->buffers[i].stream->priv);\n+\n+\t\tif (cameraStream->format != formats::MJPEG)\n+\t\t\tcontinue;\n+\n+\t\tEncoder *encoder = cameraStream->jpeg;\n+\t\tif (!encoder) {\n+\t\t\tLOG(HAL, Error) << \"Failed to identify encoder\";\n+\t\t\tcontinue;\n+\t\t}\n \n+\t\tStreamConfiguration *streamConfiguration = &config_->at(cameraStream->index);\n+\t\tStream *stream = streamConfiguration->stream();\n+\t\tFrameBuffer *buffer = request->findBuffer(stream);\n+\t\tif (!buffer) {\n+\t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\t/*\n+\t\t * \\todo Buffer mapping and compression should be moved to a\n+\t\t * separate thread.\n+\t\t */\n+\n+\t\tMappedCamera3Buffer mapped(*descriptor->buffers[i].buffer,\n+\t\t\t\t\t PROT_READ | PROT_WRITE);\n+\t\tif (!mapped.isValid()) {\n+\t\t\tLOG(HAL, Error) << \"Failed to mmap android blob buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tint jpeg_size = encoder->encode(buffer, mapped.maps()[0]);\n+\t\tif (jpeg_size < 0) {\n+\t\t\tLOG(HAL, Error) << \"Failed to encode stream image\";\n+\t\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\t/*\n+\t\t * Fill in the JPEG blob header.\n+\t\t *\n+\t\t * The mapped size of the buffer is being returned as\n+\t\t * substantially larger than the requested JPEG_MAX_SIZE\n+\t\t * (which is referenced from maxJpegBufferSize_). Utilise\n+\t\t * this static size to ensure the correct offset of the blob is\n+\t\t * determined.\n+\t\t *\n+\t\t * \\todo Investigate if the buffer size mismatch is an issue or\n+\t\t * expected behaviour.\n+\t\t */\n+\t\tuint8_t *resultPtr = mapped.maps()[0].data() +\n+\t\t\t\t maxJpegBufferSize_ -\n+\t\t\t\t sizeof(struct camera3_jpeg_blob);\n+\t\tauto *blob = reinterpret_cast<struct camera3_jpeg_blob *>(resultPtr);\n+\t\tblob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID;\n+\t\tblob->jpeg_size = jpeg_size;\n+\n+\t\t/* Update the JPEG result Metadata. */\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_SIZE,\n+\t\t\t\t\t &jpeg_size, 1);\n+\n+\t\tconst uint32_t jpeg_quality = 95;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_QUALITY,\n+\t\t\t\t\t &jpeg_quality, 1);\n+\n+\t\tconst uint32_t jpeg_orientation = 0;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_ORIENTATION,\n+\t\t\t\t\t &jpeg_orientation, 1);\n+\t}\n+\n+\t/* Prepare to call back the Android camera stack. */\n \tcamera3_capture_result_t captureResult = {};\n \tcaptureResult.frame_number = descriptor->frameNumber;\n \tcaptureResult.num_output_buffers = descriptor->numBuffers;\n@@ -1334,10 +1546,10 @@ std::unique_ptr<CameraMetadata> CameraDevice::getResultMetadata(int frame_number\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 12 entries, 36 bytes\n+\t * Currently: 18 entries, 62 bytes\n \t */\n \tstd::unique_ptr<CameraMetadata> resultMetadata =\n-\t\tstd::make_unique<CameraMetadata>(15, 50);\n+\t\tstd::make_unique<CameraMetadata>(18, 62);\n \tif (!resultMetadata->isValid()) {\n \t\tLOG(HAL, Error) << \"Failed to allocate static metadata\";\n \t\treturn nullptr;\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 4e5fb98c9a07..7be9e11977f7 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -23,15 +23,25 @@\n #include \"libcamera/internal/log.h\"\n #include \"libcamera/internal/message.h\"\n \n+#include \"jpeg/encoder.h\"\n+\n class CameraMetadata;\n \n struct CameraStream {\n+\tCameraStream(libcamera::PixelFormat, libcamera::Size);\n+\t~CameraStream();\n+\n \t/*\n \t * The index of the libcamera StreamConfiguration as added during\n \t * configureStreams(). A single libcamera Stream may be used to deliver\n \t * one or more streams to the Android framework.\n \t */\n \tunsigned int index;\n+\n+\tlibcamera::PixelFormat format;\n+\tlibcamera::Size size;\n+\n+\tEncoder *jpeg;\n };\n \n class CameraDevice : protected libcamera::Loggable\n@@ -105,6 +115,8 @@ private:\n \n \tint facing_;\n \tint orientation_;\n+\n+\tunsigned int maxJpegBufferSize_;\n };\n \n #endif /* __ANDROID_CAMERA_DEVICE_H__ */\n", "prefixes": [ "libcamera-devel", "v5", "2/2" ] }