Patch Detail
Show a patch.
GET /api/1.1/patches/9206/?format=api
{ "id": 9206, "url": "https://patchwork.libcamera.org/api/1.1/patches/9206/?format=api", "web_url": "https://patchwork.libcamera.org/patch/9206/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20200804214711.177645-14-kieran.bingham@ideasonboard.com>", "date": "2020-08-04T21:47:11", "name": "[libcamera-devel,v3,13/13] android: camera_device: Support MJPEG stream construction", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "085b2384deac790283e955fcbb2e2875095f8d85", "submitter": { "id": 4, "url": "https://patchwork.libcamera.org/api/1.1/people/4/?format=api", "name": "Kieran Bingham", "email": "kieran.bingham@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/9206/mbox/", "series": [ { "id": 1191, "url": "https://patchwork.libcamera.org/api/1.1/series/1191/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1191", "date": "2020-08-04T21:46:58", "name": "android: JPEG support", "version": 3, "mbox": "https://patchwork.libcamera.org/series/1191/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/9206/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/9206/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id B11E3BD87B\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 4 Aug 2020 21:47:26 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7C7F460548;\n\tTue, 4 Aug 2020 23:47:26 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 977A660548\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 4 Aug 2020 23:47:21 +0200 (CEST)", "from Q.local (cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net\n\t[86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 26AD5DB1;\n\tTue, 4 Aug 2020 23:47:21 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"txfCaUmO\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1596577641;\n\tbh=9Vv5QG/QPYYi8kL7gu3ejLb1PD9UXhUUmZiisXyQtx0=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=txfCaUmOPitaFrp6QEObe4PBUTIn02A99bvUXlsSNxikcL8QmxoJyHCxGlXoipC1b\n\t3SMskOt8Fb+9tHq/f+Kt4lQIAhJWZJhXR4bWyxJ1gF93lYWYncZM+Wu1lQ/JAmvAnC\n\th4eighdLOrKUxfZQLP+4TXokePRVi3wAoRarHPvM=", "From": "Kieran Bingham <kieran.bingham@ideasonboard.com>", "To": "libcamera devel <libcamera-devel@lists.libcamera.org>", "Date": "Tue, 4 Aug 2020 22:47:11 +0100", "Message-Id": "<20200804214711.177645-14-kieran.bingham@ideasonboard.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20200804214711.177645-1-kieran.bingham@ideasonboard.com>", "References": "<20200804214711.177645-1-kieran.bingham@ideasonboard.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v3 13/13] android: camera_device: Support\n\tMJPEG stream construction", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "MJPEG streams must be created referencing a libcamera stream.\nThis stream may already be provided by the request configuration,\nin which case the existing stream is utilised.\n\nIf no compatible stream is available to encode, a new stream is requested\nfrom the libcamera configuration.\n\nSigned-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\n\n---\n\nv3\n - Add max jpeg size todo\n - Fix metadata allocations\n - Use a class member to store the max jpeg buffer size\n - Remove the scoping level for jpeg blob header\n - Don't rely on the compressor pointer as a flag\n - Fix camera metadata size allocations\n---\n src/android/camera_device.cpp | 226 ++++++++++++++++++++++++++++++++--\n src/android/camera_device.h | 12 ++\n 2 files changed, 229 insertions(+), 9 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex c529246e115c..433243c3bc56 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -8,6 +8,7 @@\n #include \"camera_device.h\"\n #include \"camera_ops.h\"\n \n+#include <sys/mman.h>\n #include <tuple>\n #include <vector>\n \n@@ -22,6 +23,8 @@\n #include \"camera_metadata.h\"\n #include \"system/graphics.h\"\n \n+#include \"jpeg/compressor_jpeg.h\"\n+\n using namespace libcamera;\n \n namespace {\n@@ -129,6 +132,54 @@ const std::map<int, const Camera3Format> camera3FormatsMap = {\n \n LOG_DECLARE_CATEGORY(HAL);\n \n+class MappedCamera3Buffer : public MappedBuffer\n+{\n+public:\n+\tMappedCamera3Buffer(const buffer_handle_t camera3buffer, int flags);\n+};\n+\n+MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer,\n+\t\t\t\t\t int flags)\n+{\n+\tmaps_.reserve(camera3buffer->numFds);\n+\terror_ = 0;\n+\n+\tfor (int i = 0; i < camera3buffer->numFds; i++) {\n+\t\tif (camera3buffer->data[i] == -1)\n+\t\t\tcontinue;\n+\n+\t\toff_t length = lseek(camera3buffer->data[i], 0, SEEK_END);\n+\t\tif (length < 0) {\n+\t\t\terror_ = errno;\n+\t\t\tLOG(HAL, Error) << \"Failed to query plane length\";\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tvoid *address = mmap(nullptr, length, flags, MAP_SHARED,\n+\t\t\t\t camera3buffer->data[i], 0);\n+\t\tif (address == MAP_FAILED) {\n+\t\t\terror_ = errno;\n+\t\t\tLOG(HAL, Error) << \"Failed to mmap plane\";\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tmaps_.emplace_back(static_cast<uint8_t *>(address),\n+\t\t\t\t static_cast<size_t>(length));\n+\t}\n+\n+\tvalid_ = error_ == 0;\n+}\n+\n+CameraStream::CameraStream(PixelFormat f, Size s)\n+\t: index(-1), format(f), size(s), jpeg(nullptr)\n+{\n+}\n+\n+CameraStream::~CameraStream()\n+{\n+\tdelete jpeg;\n+};\n+\n /*\n * \\struct Camera3RequestDescriptor\n *\n@@ -167,6 +218,12 @@ CameraDevice::CameraDevice(unsigned int id, const std::shared_ptr<Camera> &camer\n \t facing_(CAMERA_FACING_FRONT), orientation_(0)\n {\n \tcamera_->requestCompleted.connect(this, &CameraDevice::requestComplete);\n+\n+\t/*\n+\t * \\todo Determine a more accurate value for this during\n+\t * streamConfiguration.\n+\t */\n+\tmax_jpeg_buffer_size_ = 13 << 20; /* 13631488 from USB HAL */\n }\n \n CameraDevice::~CameraDevice()\n@@ -417,10 +474,10 @@ std::tuple<uint32_t, uint32_t> CameraDevice::calculateStaticMetadataSize()\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 50 entries, 647 bytes of static metadata\n+\t * Currently: 51 entries, 667 bytes of static metadata\n \t */\n-\tuint32_t numEntries = 50;\n-\tuint32_t byteSize = 651;\n+\tuint32_t numEntries = 51;\n+\tuint32_t byteSize = 667;\n \n \t/*\n \t * Calculate space occupation in bytes for dynamically built metadata\n@@ -576,6 +633,12 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\t\t\t availableThumbnailSizes.data(),\n \t\t\t\t availableThumbnailSizes.size());\n \n+\t/*\n+\t * \\todo Calculate the maximum JPEG buffer size by asking the compressor\n+\t * giving the maximum frame size required.\n+\t */\n+\tstaticMetadata_->addEntry(ANDROID_JPEG_MAX_SIZE, &max_jpeg_buffer_size_, 1);\n+\n \t/* Sensor static metadata. */\n \tint32_t pixelArraySize[] = {\n \t\t2592, 1944,\n@@ -789,6 +852,7 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_CONTROL_AWB_LOCK_AVAILABLE,\n \t\tANDROID_CONTROL_AVAILABLE_MODES,\n \t\tANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES,\n+\t\tANDROID_JPEG_MAX_SIZE,\n \t\tANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE,\n \t\tANDROID_SENSOR_INFO_SENSITIVITY_RANGE,\n@@ -855,6 +919,9 @@ const camera_metadata_t *CameraDevice::getStaticMetadata()\n \t\tANDROID_SENSOR_EXPOSURE_TIME,\n \t\tANDROID_STATISTICS_LENS_SHADING_MAP_MODE,\n \t\tANDROID_STATISTICS_SCENE_FLICKER,\n+\t\tANDROID_JPEG_SIZE,\n+\t\tANDROID_JPEG_QUALITY,\n+\t\tANDROID_JPEG_ORIENTATION,\n \t};\n \tstaticMetadata_->addEntry(ANDROID_REQUEST_AVAILABLE_RESULT_KEYS,\n \t\t\t\t availableResultKeys.data(),\n@@ -1016,8 +1083,10 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t */\n \tunsigned int streamIndex = 0;\n \n+\t/* First handle all non-MJPEG streams. */\n \tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n \t\tcamera3_stream_t *stream = stream_list->streams[i];\n+\t\tSize size(stream->width, stream->height);\n \n \t\tPixelFormat format = toPixelFormat(stream->format);\n \n@@ -1031,16 +1100,71 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \t\tif (!format.isValid())\n \t\t\treturn -EINVAL;\n \n+\t\tstreams_.emplace_back(format, size);\n+\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\n+\t\t/* Defer handling of MJPEG streams until all others are known. */\n+\t\tif (format == formats::MJPEG)\n+\t\t\tcontinue;\n+\n \t\tStreamConfiguration streamConfiguration;\n \n-\t\tstreamConfiguration.size.width = stream->width;\n-\t\tstreamConfiguration.size.height = stream->height;\n+\t\tstreamConfiguration.size = size;\n \t\tstreamConfiguration.pixelFormat = format;\n \n \t\tconfig_->addConfiguration(streamConfiguration);\n \n \t\tstreams_[i].index = streamIndex++;\n-\t\tstream->priv = static_cast<void *>(&streams_[i]);\n+\t}\n+\n+\t/* Now handle MJPEG streams, adding a new stream if required. */\n+\tfor (unsigned int i = 0; i < stream_list->num_streams; ++i) {\n+\t\tcamera3_stream_t *stream = stream_list->streams[i];\n+\t\tbool match = false;\n+\n+\t\tif (streams_[i].format != formats::MJPEG)\n+\t\t\tcontinue;\n+\n+\t\t/* Search for a compatible stream */\n+\t\tfor (unsigned int j = 0; j < config_->size(); j++) {\n+\t\t\tStreamConfiguration &cfg = config_->at(j);\n+\n+\t\t\t/*\n+\t\t\t * \\todo The PixelFormat must also be compatible with\n+\t\t\t * the encoder.\n+\t\t\t */\n+\t\t\tif (cfg.size == streams_[i].size) {\n+\t\t\t\tLOG(HAL, Info) << \"Stream \" << i\n+\t\t\t\t\t << \" using libcamera stream \" << j;\n+\n+\t\t\t\tmatch = true;\n+\t\t\t\tstreams_[i].index = j;\n+\t\t\t}\n+\t\t}\n+\n+\t\t/*\n+\t\t * Without a compatible match for JPEG encoding we must\n+\t\t * introduce a new stream to satisfy the request requirements.\n+\t\t */\n+\t\tif (!match) {\n+\t\t\tStreamConfiguration streamConfiguration;\n+\n+\t\t\t/*\n+\t\t\t * \\todo: The pixelFormat should be a 'best-fit' choice\n+\t\t\t * and may require a validation cycle. This is not yet\n+\t\t\t * handled, and should be considered as part of any\n+\t\t\t * stream configuration reworks.\n+\t\t\t */\n+\t\t\tstreamConfiguration.size.width = stream->width;\n+\t\t\tstreamConfiguration.size.height = stream->height;\n+\t\t\tstreamConfiguration.pixelFormat = formats::NV12;\n+\n+\t\t\tLOG(HAL, Info) << \"Adding \" << streamConfiguration.toString()\n+\t\t\t\t << \" for MJPEG support\";\n+\n+\t\t\tconfig_->addConfiguration(streamConfiguration);\n+\t\t\tstreams_[i].index = streamIndex++;\n+\t\t}\n \t}\n \n \tswitch (config_->validate()) {\n@@ -1067,6 +1191,18 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)\n \n \t\t/* Use the bufferCount confirmed by the validation process. */\n \t\tstream->max_buffers = cfg.bufferCount;\n+\n+\t\t/*\n+\t\t * Construct a software compressor for MJPEG streams from the\n+\t\t * chosen libcamera source stream.\n+\t\t */\n+\t\tif (cameraStream->format == formats::MJPEG) {\n+\t\t\tcameraStream->jpeg = new CompressorJPEG();\n+\t\t\tcameraStream->jpeg->configure(cfg);\n+\t\t} else {\n+\t\t\t/* Either construct this correctly, or use a better interface */\n+\t\t\tcameraStream->jpeg = nullptr;\n+\t\t}\n \t}\n \n \t/*\n@@ -1161,6 +1297,10 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tdescriptor->buffers[i].stream = camera3Buffers[i].stream;\n \t\tdescriptor->buffers[i].buffer = camera3Buffers[i].buffer;\n \n+\t\t/* Software streams are handled after hardware streams complete. */\n+\t\tif (cameraStream->format == formats::MJPEG)\n+\t\t\tcontinue;\n+\n \t\t/*\n \t\t * Create a libcamera buffer using the dmabuf descriptors of\n \t\t * the camera3Buffer for each stream. The FrameBuffer is\n@@ -1217,8 +1357,76 @@ void CameraDevice::requestComplete(Request *request)\n \tresultMetadata = getResultMetadata(descriptor->frameNumber,\n \t\t\t\t\t buffer->metadata().timestamp);\n \n-\t/* Prepare to call back the Android camera stack. */\n+\t/* Handle any JPEG compression */\n+\tfor (unsigned int i = 0; i < descriptor->numBuffers; ++i) {\n+\t\tCameraStream *cameraStream =\n+\t\t\tstatic_cast<CameraStream *>(descriptor->buffers[i].stream->priv);\n+\n+\t\tif (cameraStream->format != formats::MJPEG)\n+\t\t\tcontinue;\n+\n+\t\tCompressor *compressor = cameraStream->jpeg;\n+\t\t/* Only handle streams with compression enabled. */\n+\t\tif (!compressor)\n+\t\t\tcontinue;\n+\n+\t\tStreamConfiguration *streamConfiguration = &config_->at(cameraStream->index);\n+\t\tStream *stream = streamConfiguration->stream();\n+\t\tFrameBuffer *buffer = request->findBuffer(stream);\n+\t\tif (!buffer) {\n+\t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tMappedCamera3Buffer mapped(*descriptor->buffers[i].buffer, PROT_READ | PROT_WRITE);\n+\t\tif (!mapped.isValid()) {\n+\t\t\tLOG(HAL, Error) << \"Failed to mmap android blob buffer\";\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tCompressedImage output;\n+\t\toutput.data = static_cast<unsigned char *>(mapped.maps()[0].data());\n+\t\toutput.length = mapped.maps()[0].size();\n+\n+\t\tint ret = compressor->compress(buffer, &output);\n+\t\tif (ret) {\n+\t\t\tLOG(HAL, Error) << \"Failed to compress stream image\";\n+\t\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t}\n \n+\t\t/*\n+\t\t * Fill in the JPEG blob header.\n+\t\t *\n+\t\t * The mapped size of the buffer is being returned as\n+\t\t * substantially larger than the requested JPEG_MAX_SIZE\n+\t\t * (which is referenced from max_jpeg_buffer_size_). Utilise\n+\t\t * this static size to ensure the correct offset of the blob is\n+\t\t * determined.\n+\t\t *\n+\t\t * \\todo Investigate if the buffer size mismatch is an issue or\n+\t\t * expected behaviour.\n+\t\t */\n+\t\tuint8_t *resultPtr = mapped.maps()[0].data() +\n+\t\t\t\t max_jpeg_buffer_size_ -\n+\t\t\t\t sizeof(struct camera3_jpeg_blob);\n+\t\tauto *blob = reinterpret_cast<struct camera3_jpeg_blob *>(resultPtr);\n+\t\tblob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID;\n+\t\tblob->jpeg_size = output.length;\n+\n+\t\t/* Update the JPEG result Metadata. */\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_SIZE,\n+\t\t\t\t\t &output.length, 1);\n+\n+\t\tconst uint32_t jpeg_quality = 95;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_QUALITY,\n+\t\t\t\t\t &jpeg_quality, 1);\n+\n+\t\tconst uint32_t jpeg_orientation = 0;\n+\t\tresultMetadata->addEntry(ANDROID_JPEG_ORIENTATION,\n+\t\t\t\t\t &jpeg_orientation, 1);\n+\t}\n+\n+\t/* Prepare to call back the Android camera stack. */\n \tcamera3_capture_result_t captureResult = {};\n \tcaptureResult.frame_number = descriptor->frameNumber;\n \tcaptureResult.num_output_buffers = descriptor->numBuffers;\n@@ -1298,10 +1506,10 @@ std::unique_ptr<CameraMetadata> CameraDevice::getResultMetadata(int frame_number\n {\n \t/*\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 12 entries, 36 bytes\n+\t * Currently: 18 entries, 62 bytes\n \t */\n \tstd::unique_ptr<CameraMetadata> resultMetadata =\n-\t\tstd::make_unique<CameraMetadata>(15, 50);\n+\t\tstd::make_unique<CameraMetadata>(18, 62);\n \tif (!resultMetadata->isValid()) {\n \t\tLOG(HAL, Error) << \"Failed to allocate static metadata\";\n \t\treturn nullptr;\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 00472c219388..cfec5abeffa1 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -23,15 +23,25 @@\n #include \"libcamera/internal/log.h\"\n #include \"libcamera/internal/message.h\"\n \n+#include \"jpeg/compressor_jpeg.h\"\n+\n class CameraMetadata;\n \n struct CameraStream {\n+\tCameraStream(libcamera::PixelFormat, libcamera::Size);\n+\t~CameraStream();\n+\n \t/*\n \t * The index of the libcamera StreamConfiguration as added during\n \t * configureStreams(). A single libcamera Stream may be used to deliver\n \t * one or more streams to the Android framework.\n \t */\n \tunsigned int index;\n+\n+\tlibcamera::PixelFormat format;\n+\tlibcamera::Size size;\n+\n+\tCompressorJPEG *jpeg;\n };\n \n class CameraDevice : protected libcamera::Loggable\n@@ -104,6 +114,8 @@ private:\n \n \tint facing_;\n \tint orientation_;\n+\n+\tunsigned int max_jpeg_buffer_size_;\n };\n \n #endif /* __ANDROID_CAMERA_DEVICE_H__ */\n", "prefixes": [ "libcamera-devel", "v3", "13/13" ] }