Patch Detail
Show a patch.
GET /api/1.1/patches/14189/?format=api
{ "id": 14189, "url": "https://patchwork.libcamera.org/api/1.1/patches/14189/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14189/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211020104212.121743-2-umang.jain@ideasonboard.com>", "date": "2021-10-20T10:42:09", "name": "[libcamera-devel,v5,1/4] android: Notify post processing completion via a signal", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "36fac9aa33a40d077a49cae0b2c695f4683cee2d", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/14189/mbox/", "series": [ { "id": 2640, "url": "https://patchwork.libcamera.org/api/1.1/series/2640/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2640", "date": "2021-10-20T10:42:08", "name": "Async Post Processor", "version": 5, "mbox": "https://patchwork.libcamera.org/series/2640/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14189/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14189/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id AE842BF415\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 20 Oct 2021 10:42:23 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3C3A168F5C;\n\tWed, 20 Oct 2021 12:42:23 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 450D568F58\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 20 Oct 2021 12:42:21 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.185])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 40DE62A5;\n\tWed, 20 Oct 2021 12:42:20 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"sKOZbJlI\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1634726541;\n\tbh=1n5eo1ItDKoPwgz5CdzMWKGu7I6QstwOrPYxpoX+msc=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=sKOZbJlIMLwmUqFzjrLxI6fa11HD4iGK7RJpkVK95eVBJJGhnmWCfRCxWU7AFTcbL\n\tFP4D9S565JnugL6JV7i+3N/2A2xWzq63aFyXAzCwB0b79RIjaaRnTP/lHZ+dTG0Ych\n\t8/akaMJDIro4O4dP9udRQi2UD5kdlfaa4fiARmmI=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Wed, 20 Oct 2021 16:12:09 +0530", "Message-Id": "<20211020104212.121743-2-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20211020104212.121743-1-umang.jain@ideasonboard.com>", "References": "<20211020104212.121743-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v5 1/4] android: Notify post processing\n\tcompletion via a signal", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Notify that the post processing for a request has been completed,\nvia a signal. A pointer to the descriptor which is tracking the\ncapture request is emitted along with the status of post processed\nbuffer. The function CameraDevice::streamProcessingComplete() will\nfinally set the status on the request descriptor and send capture\nresults back to the framework accordingly.\n\nWe also need to save a pointer to any internal buffers that might have\nbeen allocated by CameraStream. The buffer should be returned back to\nCameraStream just before capture results are sent.\n\nA streamProcessMutex_ has been introduced here itself, which will be\napplicable to guard access to descriptor->buffers_ when post-processing\nis moved to be asynchronous in subsequent commits.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/android/camera_device.cpp | 92 +++++++++++++++++++++---\n src/android/camera_device.h | 7 ++\n src/android/camera_request.h | 4 ++\n src/android/camera_stream.cpp | 13 ++++\n src/android/jpeg/post_processor_jpeg.cpp | 2 +\n src/android/post_processor.h | 9 +++\n src/android/yuv/post_processor_yuv.cpp | 10 ++-\n 7 files changed, 125 insertions(+), 12 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex 806b4090..541c2c81 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -1117,6 +1117,15 @@ void CameraDevice::requestComplete(Request *request)\n \t}\n \n \t/* Handle post-processing. */\n+\tbool needsPostProcessing = false;\n+\tCamera3RequestDescriptor::Status processingStatus =\n+\t\tCamera3RequestDescriptor::Status::Pending;\n+\t/*\n+\t * \\todo Apply streamsProcessMutex_ when post-processing is adapted to run\n+\t * asynchronously. If we introduce the streamsProcessMutex_ right away, the\n+\t * lock will be held forever since it is synchronous at this point\n+\t * (see streamProcessingComplete).\n+\t */\n \tfor (auto &buffer : descriptor->buffers_) {\n \t\tCameraStream *stream = buffer.stream;\n \n@@ -1132,22 +1141,27 @@ void CameraDevice::requestComplete(Request *request)\n \t\t\tcontinue;\n \t\t}\n \n-\t\tint ret = stream->process(*src, buffer, descriptor);\n-\n-\t\t/*\n-\t\t * Return the FrameBuffer to the CameraStream now that we're\n-\t\t * done processing it.\n-\t\t */\n \t\tif (stream->type() == CameraStream::Type::Internal)\n-\t\t\tstream->putBuffer(src);\n+\t\t\tbuffer.internalBuffer = src;\n \n+\t\tneedsPostProcessing = true;\n+\t\tint ret = stream->process(*src, buffer, descriptor);\n \t\tif (ret) {\n-\t\t\tbuffer.status = Camera3RequestDescriptor::Status::Error;\n-\t\t\tnotifyError(descriptor->frameNumber_, stream->camera3Stream(),\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\t\t\tsetBufferStatus(stream, buffer, descriptor,\n+\t\t\t\t\tCamera3RequestDescriptor::Status::Error);\n+\t\t\tprocessingStatus = Camera3RequestDescriptor::Status::Error;\n \t\t}\n \t}\n \n+\tif (needsPostProcessing) {\n+\t\tif (processingStatus == Camera3RequestDescriptor::Status::Error) {\n+\t\t\tdescriptor->status_ = processingStatus;\n+\t\t\tsendCaptureResults();\n+\t\t}\n+\n+\t\treturn;\n+\t}\n+\n \tdescriptor->status_ = Camera3RequestDescriptor::Status::Success;\n \tsendCaptureResults();\n }\n@@ -1206,6 +1220,64 @@ void CameraDevice::sendCaptureResults()\n \t}\n }\n \n+void CameraDevice::setBufferStatus(CameraStream *cameraStream,\n+\t\t\t\t Camera3RequestDescriptor::StreamBuffer &buffer,\n+\t\t\t\t Camera3RequestDescriptor *request,\n+\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\t/*\n+\t * Return the FrameBuffer to the CameraStream now that we're\n+\t * done processing it.\n+\t */\n+\tif (cameraStream->type() == CameraStream::Type::Internal)\n+\t\tcameraStream->putBuffer(buffer.internalBuffer);\n+\n+\tbuffer.status = status;\n+\tif (status != Camera3RequestDescriptor::Status::Success)\n+\t\tnotifyError(request->frameNumber_, buffer.stream->camera3Stream(),\n+\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+}\n+\n+void CameraDevice::streamProcessingComplete(CameraStream *cameraStream,\n+\t\t\t\t\t Camera3RequestDescriptor *request,\n+\t\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\tMutexLocker locker(request->streamsProcessMutex_);\n+\tfor (auto &buffer : request->buffers_) {\n+\t\tif (buffer.stream != cameraStream)\n+\t\t\tcontinue;\n+\n+\t\tsetBufferStatus(cameraStream, buffer, request, status);\n+\t}\n+\n+\tbool hasPostProcessingErrors = false;\n+\tfor (auto &buffer : request->buffers_) {\n+\t\tif (cameraStream->type() == CameraStream::Type::Direct)\n+\t\t\tcontinue;\n+\n+\t\t/*\n+\t\t * Other eligible buffers might be waiting to get post-processed.\n+\t\t * So wait for their turn before sendCaptureResults() for the\n+\t\t * descriptor.\n+\t\t */\n+\t\tif (buffer.status == Camera3RequestDescriptor::Status::Pending)\n+\t\t\treturn;\n+\n+\t\tif (!hasPostProcessingErrors &&\n+\t\t buffer.status == Camera3RequestDescriptor::Status::Error)\n+\t\t\thasPostProcessingErrors = true;\n+\t}\n+\n+\tif (hasPostProcessingErrors)\n+\t\trequest->status_ = Camera3RequestDescriptor::Status::Error;\n+\telse\n+\t\trequest->status_ = Camera3RequestDescriptor::Status::Success;\n+\n+\tlocker.unlock();\n+\n+\tsendCaptureResults();\n+}\n+\n std::string CameraDevice::logPrefix() const\n {\n \treturn \"'\" + camera_->id() + \"'\";\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 863cf414..1ef933da 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -66,6 +66,9 @@ public:\n \tint configureStreams(camera3_stream_configuration_t *stream_list);\n \tint processCaptureRequest(camera3_capture_request_t *request);\n \tvoid requestComplete(libcamera::Request *request);\n+\tvoid streamProcessingComplete(CameraStream *cameraStream,\n+\t\t\t\t Camera3RequestDescriptor *request,\n+\t\t\t\t Camera3RequestDescriptor::Status status);\n \n protected:\n \tstd::string logPrefix() const override;\n@@ -94,6 +97,10 @@ private:\n \t\t\t camera3_error_msg_code code) const;\n \tint processControls(Camera3RequestDescriptor *descriptor);\n \tvoid sendCaptureResults();\n+\tvoid setBufferStatus(CameraStream *cameraStream,\n+\t\t\t Camera3RequestDescriptor::StreamBuffer &buffer,\n+\t\t\t Camera3RequestDescriptor *request,\n+\t\t\t Camera3RequestDescriptor::Status status);\n \tstd::unique_ptr<CameraMetadata> getResultMetadata(\n \t\tconst Camera3RequestDescriptor &descriptor) const;\n \ndiff --git a/src/android/camera_request.h b/src/android/camera_request.h\nindex 05dabf89..3a2774e0 100644\n--- a/src/android/camera_request.h\n+++ b/src/android/camera_request.h\n@@ -8,6 +8,7 @@\n #define __ANDROID_CAMERA_REQUEST_H__\n \n #include <memory>\n+#include <mutex>\n #include <vector>\n \n #include <libcamera/base/class.h>\n@@ -37,6 +38,7 @@ public:\n \t\tstd::unique_ptr<libcamera::FrameBuffer> frameBuffer;\n \t\tint fence;\n \t\tStatus status;\n+\t\tlibcamera::FrameBuffer *internalBuffer = nullptr;\n \t};\n \n \tCamera3RequestDescriptor(libcamera::Camera *camera,\n@@ -47,6 +49,8 @@ public:\n \n \tuint32_t frameNumber_ = 0;\n \n+\t/* Protects buffers_ for post-processing streams. */\n+\tstd::mutex streamsProcessMutex_;\n \tstd::vector<StreamBuffer> buffers_;\n \n \tCameraMetadata settings_;\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex f44a2717..04cbef8c 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -22,6 +22,7 @@\n #include \"camera_capabilities.h\"\n #include \"camera_device.h\"\n #include \"camera_metadata.h\"\n+#include \"post_processor.h\"\n \n using namespace libcamera;\n \n@@ -97,6 +98,18 @@ int CameraStream::configure()\n \t\tint ret = postProcessor_->configure(configuration(), output);\n \t\tif (ret)\n \t\t\treturn ret;\n+\n+\t\tpostProcessor_->processComplete.connect(\n+\t\t\tthis, [&](Camera3RequestDescriptor *request, PostProcessor::Status status) {\n+\t\t\t\tCamera3RequestDescriptor::Status bufferStatus =\n+\t\t\t\t\tCamera3RequestDescriptor::Status::Error;\n+\n+\t\t\t\tif (status == PostProcessor::Status::Success)\n+\t\t\t\t\tbufferStatus = Camera3RequestDescriptor::Status::Success;\n+\n+\t\t\t\tcameraDevice_->streamProcessingComplete(this, request,\n+\t\t\t\t\t\t\t\t\tbufferStatus);\n+\t\t\t});\n \t}\n \n \tif (type_ == Type::Internal) {\ndiff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\nindex 699576ef..a001fede 100644\n--- a/src/android/jpeg/post_processor_jpeg.cpp\n+++ b/src/android/jpeg/post_processor_jpeg.cpp\n@@ -198,6 +198,7 @@ int PostProcessorJpeg::process(const FrameBuffer &source,\n \t\t\t\t\t exif.data(), quality);\n \tif (jpeg_size < 0) {\n \t\tLOG(JPEG, Error) << \"Failed to encode stream image\";\n+\t\tprocessComplete.emit(request, PostProcessor::Status::Error);\n \t\treturn jpeg_size;\n \t}\n \n@@ -211,6 +212,7 @@ int PostProcessorJpeg::process(const FrameBuffer &source,\n \n \t/* Update the JPEG result Metadata. */\n \tresultMetadata->addEntry(ANDROID_JPEG_SIZE, jpeg_size);\n+\tprocessComplete.emit(request, PostProcessor::Status::Success);\n \n \treturn 0;\n }\ndiff --git a/src/android/post_processor.h b/src/android/post_processor.h\nindex 27eaef88..14f5e8c7 100644\n--- a/src/android/post_processor.h\n+++ b/src/android/post_processor.h\n@@ -7,6 +7,8 @@\n #ifndef __ANDROID_POST_PROCESSOR_H__\n #define __ANDROID_POST_PROCESSOR_H__\n \n+#include <libcamera/base/signal.h>\n+\n #include <libcamera/framebuffer.h>\n #include <libcamera/stream.h>\n \n@@ -17,6 +19,11 @@ class Camera3RequestDescriptor;\n class PostProcessor\n {\n public:\n+\tenum class Status {\n+\t\tError,\n+\t\tSuccess\n+\t};\n+\n \tvirtual ~PostProcessor() = default;\n \n \tvirtual int configure(const libcamera::StreamConfiguration &inCfg,\n@@ -24,6 +31,8 @@ public:\n \tvirtual int process(const libcamera::FrameBuffer &source,\n \t\t\t CameraBuffer *destination,\n \t\t\t Camera3RequestDescriptor *request) = 0;\n+\n+\tlibcamera::Signal<Camera3RequestDescriptor *, Status> processComplete;\n };\n \n #endif /* __ANDROID_POST_PROCESSOR_H__ */\ndiff --git a/src/android/yuv/post_processor_yuv.cpp b/src/android/yuv/post_processor_yuv.cpp\nindex 8110a1f1..fd364741 100644\n--- a/src/android/yuv/post_processor_yuv.cpp\n+++ b/src/android/yuv/post_processor_yuv.cpp\n@@ -51,14 +51,17 @@ int PostProcessorYuv::configure(const StreamConfiguration &inCfg,\n \n int PostProcessorYuv::process(const FrameBuffer &source,\n \t\t\t CameraBuffer *destination,\n-\t\t\t [[maybe_unused]] Camera3RequestDescriptor *request)\n+\t\t\t Camera3RequestDescriptor *request)\n {\n-\tif (!isValidBuffers(source, *destination))\n+\tif (!isValidBuffers(source, *destination)) {\n+\t\tprocessComplete.emit(request, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n+\t}\n \n \tconst MappedFrameBuffer sourceMapped(&source, MappedFrameBuffer::MapFlag::Read);\n \tif (!sourceMapped.isValid()) {\n \t\tLOG(YUV, Error) << \"Failed to mmap camera frame buffer\";\n+\t\tprocessComplete.emit(request, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n@@ -76,9 +79,12 @@ int PostProcessorYuv::process(const FrameBuffer &source,\n \t\t\t\t libyuv::FilterMode::kFilterBilinear);\n \tif (ret) {\n \t\tLOG(YUV, Error) << \"Failed NV12 scaling: \" << ret;\n+\t\tprocessComplete.emit(request, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n+\tprocessComplete.emit(request, PostProcessor::Status::Success);\n+\n \treturn 0;\n }\n \n", "prefixes": [ "libcamera-devel", "v5", "1/4" ] }