Patch Detail
Show a patch.
GET /api/patches/9152/?format=api
{ "id": 9152, "url": "https://patchwork.libcamera.org/api/patches/9152/?format=api", "web_url": "https://patchwork.libcamera.org/patch/9152/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20200803161816.107113-13-kieran.bingham@ideasonboard.com>", "date": "2020-08-03T16:18:16", "name": "[libcamera-devel,v2,12/12] android: camera_device: Support MJPEG stream construction", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "bafbcbf889dfd83eb87ee52c1be240a10a90e73e", "submitter": { "id": 4, "url": "https://patchwork.libcamera.org/api/people/4/?format=api", "name": "Kieran Bingham", "email": "kieran.bingham@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/9152/mbox/", "series": [ { "id": 1183, "url": "https://patchwork.libcamera.org/api/series/1183/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1183", "date": "2020-08-03T16:18:04", "name": "android: jpeg", "version": 2, "mbox": "https://patchwork.libcamera.org/series/1183/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/9152/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/9152/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E61B7BD86F\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 3 Aug 2020 16:18:30 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B00D161A12;\n\tMon, 3 Aug 2020 18:18:30 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4BA6B61B35\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 3 Aug 2020 18:18:27 +0200 (CEST)", "from Q.local (cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net\n\t[86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id DC345112C;\n\tMon, 3 Aug 2020 18:18:26 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"UHERhVzu\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1596471507;\n\tbh=ewZFYXoI8DWP3N+YUCUoCaloJSkAZAnh3QdATsykt4Q=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=UHERhVzuq8Oj7mo5X1KEDonhwMY0y8aA1bZ8EW31J1eyBZnb7kVoqbdklj98B7Mqd\n\tOhQclHYeAngLGXTYe3o1ptfaan9gbWYmj1fKYNgk6x5lHgj99+RNAjXI+I6XfWxcP2\n\tnVavpMqPuWDUi3GmWxvWnbdyqn1BUuTku8E1uOQs=", "From": "Kieran Bingham <kieran.bingham@ideasonboard.com>", "To": "libcamera devel <libcamera-devel@lists.libcamera.org>", "Date": "Mon, 3 Aug 2020 17:18:16 +0100", "Message-Id": "<20200803161816.107113-13-kieran.bingham@ideasonboard.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20200803161816.107113-1-kieran.bingham@ideasonboard.com>", "References": "<20200803161816.107113-1-kieran.bingham@ideasonboard.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v2 12/12] android: camera_device: Support\n\tMJPEG stream construction", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "MJPEG streams must be created referencing a libcamera stream.\nThis stream may already be provided by the request configuration,\nin which case the existing stream is utilised.\n\nIf no compatible stream is available to encode, a new stream is requested\nfrom the libcamera configuration.\n\nSigned-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\n---\n\nThis is probably the main patch holding me up at the moment, as there\nare just so many bits I'm not happy with.\n\n - Clean up the \"Fill in the JPEG blob header.\"\n - Added streams for different MPEG requirements don't have their own\n buffers (only just realised that today, so I think we should just\n pull that out for a first integration)\n - JPEG compression should run in a thread.\n - I really dislike the current mapping between camera3_stream to\n libcamera stream with the index\n - The JPEG compressor is not currently destructed.\n I was going to handle that by moving the CameraStream to a class\n which would then destroy the compressor in it's destructor - but that\n hasn't gone down that path, maybe it still should.\n - Metadata additions are horrible to keep in sync (especially through\n rebases)\n - More stuff ;-)\n\nAnyway, here's the current WIP - and it works ... so hey that's a bonus\n ;-)\n\n\n\n\n src/android/camera_device.cpp | 202 +++++++++++++++++++++++++++++++++-\n src/android/camera_device.h | 8 ++\n 2 files changed, 204 insertions(+), 6 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex e23ab055d012..42b08cfc5fed 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -8,6 +8,7 @@\n #include \"camera_device.h\"\n #include \"camera_ops.h\"\n \n+#include <sys/mman.h>\n #include <tuple>\n #include <vector>\n \n@@ -21,6 +22,8 @@\n #include \"camera_metadata.h\"\n #include \"system/graphics.h\"\n \n+#include \"jpeg/compressor_jpeg.h\"\n+\n using namespace libcamera;\n \n namespace {\n@@ -80,6 +83,40 @@ const std::map<int, const Camera3Format> camera3FormatsMap = {\n \n LOG_DECLARE_CATEGORY(HAL);\n \n+class MappedCamera3Buffer : public MappedBuffer\n+{\n+public:\n+\tMappedCamera3Buffer(const buffer_handle_t camera3buffer, int flags)\n+\t{\n+\t\tmaps_.reserve(camera3buffer->numFds);\n+\t\terror_ = 0;\n+\n+\t\tfor (int i = 0; i < camera3buffer->numFds; i++) {\n+\t\t\tif (camera3buffer->data[i] == -1)\n+\t\t\t\tcontinue;\n+\n+\t\t\toff_t length = lseek(camera3buffer->data[i], 0, SEEK_END);\n+\t\t\tif (length < 0) {\n+\t\t\t\terror_ = errno;\n+\t\t\t\tLOG(HAL, Error) << \"Failed to query plane length\";\n+\t\t\t\tbreak;\n+\t\t\t}\n+\n+\t\t\tvoid *address = mmap(nullptr, length, flags, MAP_SHARED,\n+\t\t\t\t\t camera3buffer->data[i], 0);\n+\t\t\tif (address == MAP_FAILED) {\n+\t\t\t\terror_ = errno;\n+\t\t\t\tLOG(HAL, Error) << \"Failed to mmap plane\";\n+\t\t\t\tbreak;\n+\t\t\t}\n+\n+\t\t\tmaps_.emplace_back(static_cast<uint8_t *>(address), static_cast<size_t>(length));\n+\t\t}\n+\n+\t\tvalid_ = error_ == 0;\n+\t}\n+};\n+\n /*\n * \\struct Camera3RequestDescriptor\n *\n@@ -368,9 +405,9 @@ std::tuple<uint32_t, uint32_t> CameraDevice::calculateStaticMetadataSize()\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 50 entries, 647 bytes of static metadata\n+\t * Currently: 51 entries, 651 bytes of static metadata\n \t */\n-\tuint32_t numEntries = 50;\n+\tuint32_t numEntries = 51;\n \tuint32_t byteSize = 651;\n \n \t/*\n@@ -527,6 +564,9 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\t\t\t availableThumbnailSizes.data(),\n \t\t\t\t availableThumbnailSizes.size());\n \n+\tint32_t jpegMaxSize = 13 << 20; /* 13631488 from USB HAL */\n+\tstaticMetadata_->addEntry(ANDROID_JPEG_MAX_SIZE, &jpegMaxSize, 1);\n+\n \t/* Sensor static metadata. */\n \tint32_t pixelArraySize[] = {\n \t\t2592, 1944,\n@@ -729,6 +769,7 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_CONTROL_AWB_LOCK_AVAILABLE,\n \t\tANDROID_CONTROL_AVAILABLE_MODES,\n \t\tANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES,\n+\t\tANDROID_JPEG_MAX_SIZE,\n \t\tANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_SENSITIVITY_RANGE,\n@@ -795,6 +836,9 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_SENSOR_EXPOSURE_TIME,\n \t\tANDROID_STATISTICS_LENS_SHADING_MAP_MODE,\n \t\tANDROID_STATISTICS_SCENE_FLICKER,\n+\t\tANDROID_JPEG_SIZE,\n+\t\tANDROID_JPEG_QUALITY,\n+\t\tANDROID_JPEG_ORIENTATION,\n \t};\n \tstaticMetadata_->addEntry(ANDROID_REQUEST_AVAILABLE_RESULT_KEYS,\n \t\t\t\t availableResultKeys.data(),\n@@ -956,6 +1000,7 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t */\n \tunsigned int streamIndex = 0;\n \n+\t/* First handle all non-MJPEG streams */\n \tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n \t\tcamera3_stream_t *stream = stream_list->streams[i];\n \n@@ -971,6 +1016,18 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t\tif (!format.isValid())\n \t\t\treturn -EINVAL;\n \n+\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\t\tstreams_[i].format = format;\n+\t\tstreams_[i].size = Size(stream->width, stream->height);\n+\n+\t\t/* Defer handling of MJPEG streams until all others are known. */\n+\t\tif (format == formats::MJPEG) {\n+\t\t\tLOG(HAL, Info) << \"Handling MJPEG stream\";\n+\n+\t\t\tstreams_[i].index = -1;\n+\t\t\tcontinue;\n+\t\t}\n+\n \t\tStreamConfiguration streamConfiguration;\n \n \t\tstreamConfiguration.size.width = stream->width;\n@@ -980,7 +1037,61 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t\tconfig_->addConfiguration(streamConfiguration);\n \n \t\tstreams_[i].index = streamIndex++;\n-\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\t}\n+\n+\t/* Now handle MJPEG streams, adding a new stream if required. */\n+\tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n+\t\tcamera3_stream_t *stream = stream_list->streams[i];\n+\t\tCameraStream *cameraStream = &streams_[i];\n+\n+\t\t/* Only process MJPEG streams */\n+\t\tif (cameraStream->format != formats::MJPEG)\n+\t\t\tcontinue;\n+\n+\t\tbool match = false;\n+\n+\t\t/* Search for a compatible stream */\n+\t\tfor (unsigned int j = 0; j < config_->size(); j++) {\n+\t\t\tStreamConfiguration &cfg = config_->at(j);\n+\n+\t\t\t/*\n+\t\t\t * The PixelFormat must also be compatible with the\n+\t\t\t * encoder.\n+\t\t\t */\n+\t\t\tif (cfg.size == cameraStream->size) {\n+\t\t\t\tLOG(HAL, Info)\n+\t\t\t\t\t<< \"Stream \" << i\n+\t\t\t\t\t<< \" using libcamera stream \"\n+\t\t\t\t\t<< j;\n+\n+\t\t\t\tmatch = true;\n+\t\t\t\tcameraStream->index = j;\n+\t\t\t}\n+\t\t}\n+\n+\t\t/*\n+\t\t * Without a compatible match for JPEG encoding we must\n+\t\t * introduce a new stream to satisfy the request requirements.\n+\t\t */\n+\t\tif (!match) {\n+\t\t\tStreamConfiguration streamConfiguration;\n+\n+\t\t\t/*\n+\t\t\t * \\todo: The pixelFormat should be a 'best-fit' choice\n+\t\t\t * and may require a validation cycle. This is not yet\n+\t\t\t * handled, and should be considered as part of any\n+\t\t\t * stream configuration reworks.\n+\t\t\t */\n+\t\t\tstreamConfiguration.size.width = stream->width;\n+\t\t\tstreamConfiguration.size.height = stream->height;\n+\t\t\tstreamConfiguration.pixelFormat = formats::NV12;\n+\n+\t\t\tLOG(HAL, Info) << \"Adding \" << streamConfiguration.toString()\n+\t\t\t\t << \" for MJPEG support\";\n+\n+\t\t\tconfig_->addConfiguration(streamConfiguration);\n+\t\t\tstreams_[i].index = streamIndex++;\n+\t\t}\n \t}\n \n \tswitch (config_->validate()) {\n@@ -1007,6 +1118,18 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \n \t\t/* Use the bufferCount confirmed by the validation process. */\n \t\tstream->max_buffers = cfg.bufferCount;\n+\n+\t\t/*\n+\t\t * Construct a software compressor for MJPEG streams from the\n+\t\t * chosen libcamera source stream.\n+\t\t */\n+\t\tif (cameraStream->format == formats::MJPEG) {\n+\t\t\tcameraStream->jpeg = new CompressorJPEG();\n+\t\t\tcameraStream->jpeg->configure(cfg);\n+\t\t} else {\n+\t\t\t/* Either construct this correctly, or use a better interface */\n+\t\t\tcameraStream->jpeg = nullptr;\n+\t\t}\n \t}\n \n \t/*\n@@ -1101,6 +1224,10 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tdescriptor->buffers[i].stream = camera3Buffers[i].stream;\n \t\tdescriptor->buffers[i].buffer = camera3Buffers[i].buffer;\n \n+\t\t/* Software streams are handled after hardware streams complete. */\n+\t\tif (cameraStream->format == formats::MJPEG)\n+\t\t\tcontinue;\n+\n \t\t/*\n \t\t * Create a libcamera buffer using the dmabuf descriptors of\n \t\t * the camera3Buffer for each stream. The FrameBuffer is\n@@ -1157,9 +1284,72 @@ void CameraDevice::requestComplete(Request *request)\n \tresultMetadata = getResultMetadata(descriptor->frameNumber,\n \t\t\t\t\t buffer->metadata().timestamp);\n \n-\t/* Prepare to call back the Android camera stack. */\n+\t/* Handle any JPEG compression */\n+\tfor (unsigned int i = 0; i < descriptor->numBuffers; ++i) {\n+\t\tCameraStream *cameraStream =\n+\t\t\tstatic_cast<CameraStream *>(descriptor->buffers[i].stream->priv);\n+\t\tCompressor *compressor = cameraStream->jpeg;\n \n+\t\t/* Only handle streams with compression enabled. */\n+\t\tif (!compressor)\n+\t\t\tcontinue;\n \n+\t\tStreamConfiguration *streamConfiguration = &config_->at(cameraStream->index);\n+\t\tStream *stream = streamConfiguration->stream();\n+\t\tFrameBuffer *buffer = request->findBuffer(stream);\n+\t\tif (!buffer) {\n+\t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tMappedCamera3Buffer mapped(*descriptor->buffers[i].buffer, PROT_READ | PROT_WRITE);\n+\t\tif (!mapped.isValid()) {\n+\t\t\tLOG(HAL, Error) << \"Failed to mmap android blob buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tCompressedImage output;\n+\t\toutput.data = static_cast<unsigned char *>(mapped.maps()[0].data());\n+\t\toutput.length = mapped.maps()[0].size();\n+\n+\t\tint ret = compressor->compress(buffer, &output);\n+\t\tif (ret) {\n+\t\t\tLOG(HAL, Error) << \"Failed to compress stream image\";\n+\t\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t}\n+\n+\t\t/* Fill in the JPEG blob header. */\n+\t\t{\n+\t\t\t/* The mapped size of the buffer is being returned as substantially larger\n+\t\t\t * than the requested JPEG_MAX_SIZE (re-defined here).\n+\t\t\t * Utilise this static size as a workaround to ensure the correct offset\n+\t\t\t * of the blob is determined.\n+\t\t\t *\n+\t\t\t * \\todo Investigate and fix the root cause of this.\n+\t\t\t */\n+\t\t\tint32_t max_size = { 13 << 20 };\n+\t\t\tuint8_t *resultPtr = static_cast<uint8_t *>(mapped.maps()[0].data()) +\n+\t\t\t\t\t max_size - sizeof(struct camera3_jpeg_blob);\n+\t\t\tauto *blob = reinterpret_cast<struct camera3_jpeg_blob *>(resultPtr);\n+\n+\t\t\tblob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID;\n+\t\t\tblob->jpeg_size = output.length;\n+\t\t}\n+\n+\t\t/* Update the JPEG result Metadata. */\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_SIZE,\n+\t\t\t\t\t &output.length, 1);\n+\n+\t\tconst uint32_t jpeg_quality = 95;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_QUALITY,\n+\t\t\t\t\t &jpeg_quality, 1);\n+\n+\t\tconst uint32_t jpeg_orientation = 0;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_ORIENTATION,\n+\t\t\t\t\t &jpeg_orientation, 1);\n+\t}\n+\n+\t/* Prepare to call back the Android camera stack. */\n \tcamera3_capture_result_t captureResult = {};\n \tcaptureResult.frame_number = descriptor->frameNumber;\n \tcaptureResult.num_output_buffers = descriptor->numBuffers;\n@@ -1240,10 +1430,10 @@ std::unique_ptr<CameraMetadata> CameraDevice::getResultMetadata(int frame_number\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 12 entries, 36 bytes\n+\t * Currently: 17 entries, 58 bytes\n \t */\n \tstd::unique_ptr<CameraMetadata> resultMetadata =\n-\t\tstd::make_unique<CameraMetadata>(15, 50);\n+\t\tstd::make_unique<CameraMetadata>(17, 58);\n \tif (!resultMetadata->isValid()) {\n \t\tLOG(HAL, Error) << \"Failed to allocate static metadata\";\n \t\treturn nullptr;\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 00472c219388..c9ece426a00e 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -23,6 +23,8 @@\n #include \"libcamera/internal/log.h\"\n #include \"libcamera/internal/message.h\"\n \n+#include \"jpeg/compressor_jpeg.h\"\n+\n class CameraMetadata;\n \n struct CameraStream {\n@@ -32,6 +34,12 @@ struct CameraStream {\n \t * one or more streams to the Android framework.\n \t */\n \tunsigned int index;\n+\n+\tlibcamera::PixelFormat format;\n+\tlibcamera::Size size;\n+\n+\t/* Make sure this gets destructed correctly */\n+\tCompressorJPEG *jpeg;\n };\n \n class CameraDevice : protected libcamera::Loggable\n", "prefixes": [ "libcamera-devel", "v2", "12/12" ] }