Patch Detail
Show a patch.
GET /api/1.1/patches/14299/?format=api
{ "id": 14299, "url": "https://patchwork.libcamera.org/api/1.1/patches/14299/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14299/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211025203833.122460-6-umang.jain@ideasonboard.com>", "date": "2021-10-25T20:38:31", "name": "[libcamera-devel,v7,5/7] android: Track and notify post processing of streams", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "7a01630bd1a7217971a458b03b3f99d8fcfb9503", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": { "id": 12, "url": "https://patchwork.libcamera.org/api/1.1/users/12/?format=api", "username": "uajain", "first_name": "Umang", "last_name": "Jain", "email": "umang.jain@ideasonboard.com" }, "mbox": "https://patchwork.libcamera.org/patch/14299/mbox/", "series": [ { "id": 2653, "url": "https://patchwork.libcamera.org/api/1.1/series/2653/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2653", "date": "2021-10-25T20:38:26", "name": "Async Post Processor", "version": 7, "mbox": "https://patchwork.libcamera.org/series/2653/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14299/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14299/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 80CDDBF415\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 25 Oct 2021 20:38:55 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 4428964881;\n\tMon, 25 Oct 2021 22:38:55 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id E3EA36486E\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 25 Oct 2021 22:38:52 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.211])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 095F6119F;\n\tMon, 25 Oct 2021 22:38:51 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"LDO9NmYV\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1635194332;\n\tbh=GsUusicoSbQ0ot/rMEyORmc+Xe4rOS9e+ed3WeVBwqs=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=LDO9NmYVpOZXL9O0SRfOml5d/B3NV6zKJJLcccesVw5BYv0cqhtTgnqzewHdPv0Jj\n\tewV/saSbyeZD39EF530gdBSgRZWeyV2Kt8BDYdoOEU3GITnVLmlDqAX6YbS++JS8QN\n\t9DOS5jRz9ZLakmktflS2A8ePMcHnq7IqA2F8BYpw=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Tue, 26 Oct 2021 02:08:31 +0530", "Message-Id": "<20211025203833.122460-6-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20211025203833.122460-1-umang.jain@ideasonboard.com>", "References": "<20211025203833.122460-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v7 5/7] android: Track and notify post\n\tprocessing of streams", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Notify that the post processing for a request has been completed,\nvia a signal. The signal is emitted with a context pointer along with\nstatus of the buffer. The function CameraDevice::streamProcessingComplete()\nwill finally set the status on the request descriptor and complete the\ndescriptor if all the streams requiring post processing are completed.\nIf buffer status obtained is in error state, notify the status to the\nframework and set the overall error status on the descriptor via\nsetBufferStatus().\n\nWe need to track the number of streams requiring post-processing\nper Camera3RequestDescriptor (i.e. per capture request). Introduce\na std::map<> to track the post-processing of streams. The nodes\nare dropped from the map when a particular stream post processing\nis completed (or on error paths). A std::map is selected for tracking\npost-processing requests, since we will move post-processing to be\nasynchronous in subsequent commits. A vector or queue will not be\nsuitable then as the sequential order of post-processing completion\nof various requests won't be guaranteed then.\n\nA streamsProcessMutex_ has been introduced here as well, which will be\napplicable to guard access to descriptor's pendingStreamsToProcess_ when\npost-processing is moved to be asynchronous in subsequent commits.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/android/camera_device.cpp | 87 +++++++++++++++++-------\n src/android/camera_device.h | 4 ++\n src/android/camera_request.h | 6 ++\n src/android/camera_stream.cpp | 15 ++++\n src/android/jpeg/post_processor_jpeg.cpp | 2 +\n src/android/post_processor.h | 9 +++\n src/android/yuv/post_processor_yuv.cpp | 8 ++-\n 7 files changed, 106 insertions(+), 25 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex 9155728a..3ded0f7e 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -926,6 +926,9 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\t * Request.\n \t\t\t */\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (mapped)\";\n+\n+\t\t\tdescriptor->pendingStreamsToProcess_.insert(\n+\t\t\t\t{ cameraStream, &buffer });\n \t\t\tcontinue;\n \n \t\tcase CameraStream::Type::Direct:\n@@ -955,6 +958,9 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\tframeBuffer = cameraStream->getBuffer();\n \t\t\tbuffer.internalBuffer = frameBuffer;\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (internal)\";\n+\n+\t\t\tdescriptor->pendingStreamsToProcess_.insert(\n+\t\t\t\t{ cameraStream, &buffer });\n \t\t\tbreak;\n \t\t}\n \n@@ -1118,42 +1124,42 @@ void CameraDevice::requestComplete(Request *request)\n \t}\n \n \t/* Handle post-processing. */\n-\tfor (auto &buffer : descriptor->buffers_) {\n-\t\tCameraStream *stream = buffer.stream;\n-\n-\t\tif (stream->type() == CameraStream::Type::Direct)\n-\t\t\tcontinue;\n+\t/*\n+\t * \\todo Protect the loop below with streamProcessMutex_ when post\n+\t * processor runs asynchronously.\n+\t */\n+\tauto iter = descriptor->pendingStreamsToProcess_.begin();\n+\twhile (iter != descriptor->pendingStreamsToProcess_.end()) {\n+\t\tCameraStream *stream = iter->first;\n+\t\tCamera3RequestDescriptor::StreamBuffer *buffer = iter->second;\n \n \t\tFrameBuffer *src = request->findBuffer(stream->stream());\n \t\tif (!src) {\n \t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n-\t\t\tbuffer.status = Camera3RequestDescriptor::Status::Error;\n-\t\t\tnotifyError(descriptor->frameNumber_, stream->camera3Stream(),\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n-\t\t\tdescriptor->status_ = Camera3RequestDescriptor::Status::Error;\n+\t\t\tsetBufferStatus(*buffer, Camera3RequestDescriptor::Status::Error);\n+\t\t\titer = descriptor->pendingStreamsToProcess_.erase(iter);\n \t\t\tcontinue;\n \t\t}\n \n-\t\tbuffer.srcBuffer = src;\n-\n-\t\tint ret = stream->process(&buffer);\n-\n-\t\t/*\n-\t\t * If the framebuffer is internal to CameraStream return it back\n-\t\t * now that we're done processing it.\n-\t\t */\n-\t\tif (buffer.internalBuffer)\n-\t\t\tstream->putBuffer(buffer.internalBuffer);\n+\t\tbuffer->srcBuffer = src;\n \n+\t\t++iter;\n+\t\tint ret = stream->process(buffer);\n \t\tif (ret) {\n-\t\t\tbuffer.status = Camera3RequestDescriptor::Status::Error;\n-\t\t\tnotifyError(descriptor->frameNumber_, stream->camera3Stream(),\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n-\t\t\tdescriptor->status_ = Camera3RequestDescriptor::Status::Error;\n+\t\t\tsetBufferStatus(*buffer, Camera3RequestDescriptor::Status::Error);\n+\t\t\tdescriptor->pendingStreamsToProcess_.erase(stream);\n+\n+\t\t\t/*\n+\t\t\t * If the framebuffer is internal to CameraStream return\n+\t\t\t * it back now that we're done processing it.\n+\t\t\t */\n+\t\t\tif (buffer->internalBuffer)\n+\t\t\t\tstream->putBuffer(buffer->internalBuffer);\n \t\t}\n \t}\n \n-\tcompleteDescriptor(descriptor);\n+\tif (descriptor->pendingStreamsToProcess_.empty())\n+\t\tcompleteDescriptor(descriptor);\n }\n \n void CameraDevice::completeDescriptor(Camera3RequestDescriptor *descriptor)\n@@ -1208,6 +1214,39 @@ void CameraDevice::sendCaptureResults()\n \t}\n }\n \n+void CameraDevice::setBufferStatus(Camera3RequestDescriptor::StreamBuffer &streamBuffer,\n+\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\tstreamBuffer.status = status;\n+\tif (status != Camera3RequestDescriptor::Status::Success) {\n+\t\tnotifyError(streamBuffer.request->frameNumber_,\n+\t\t\t streamBuffer.stream->camera3Stream(),\n+\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\n+\t\t/* Also set error status on entire request descriptor. */\n+\t\tstreamBuffer.request->status_ =\n+\t\t\tCamera3RequestDescriptor::Status::Error;\n+\t}\n+}\n+\n+void CameraDevice::streamProcessingComplete(Camera3RequestDescriptor::StreamBuffer *streamBuffer,\n+\t\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\tsetBufferStatus(*streamBuffer, status);\n+\n+\t/*\n+\t * If the framebuffer is internal to CameraStream return it back now\n+\t * that we're done processing it.\n+\t */\n+\tif (streamBuffer->internalBuffer)\n+\t\tstreamBuffer->stream->putBuffer(streamBuffer->internalBuffer);\n+\n+\tCamera3RequestDescriptor *request = streamBuffer->request;\n+\tMutexLocker locker(request->streamsProcessMutex_);\n+\n+\trequest->pendingStreamsToProcess_.erase(streamBuffer->stream);\n+}\n+\n std::string CameraDevice::logPrefix() const\n {\n \treturn \"'\" + camera_->id() + \"'\";\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex e544f2bd..2a414020 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -66,6 +66,8 @@ public:\n \tint configureStreams(camera3_stream_configuration_t *stream_list);\n \tint processCaptureRequest(camera3_capture_request_t *request);\n \tvoid requestComplete(libcamera::Request *request);\n+\tvoid streamProcessingComplete(Camera3RequestDescriptor::StreamBuffer *bufferStream,\n+\t\t\t\t Camera3RequestDescriptor::Status status);\n \n protected:\n \tstd::string logPrefix() const override;\n@@ -95,6 +97,8 @@ private:\n \tint processControls(Camera3RequestDescriptor *descriptor);\n \tvoid completeDescriptor(Camera3RequestDescriptor *descriptor);\n \tvoid sendCaptureResults();\n+\tvoid setBufferStatus(Camera3RequestDescriptor::StreamBuffer &buffer,\n+\t\t\t Camera3RequestDescriptor::Status status);\n \tstd::unique_ptr<CameraMetadata> getResultMetadata(\n \t\tconst Camera3RequestDescriptor &descriptor) const;\n \ndiff --git a/src/android/camera_request.h b/src/android/camera_request.h\nindex c7fda00d..c28f7942 100644\n--- a/src/android/camera_request.h\n+++ b/src/android/camera_request.h\n@@ -7,7 +7,9 @@\n #ifndef __ANDROID_CAMERA_REQUEST_H__\n #define __ANDROID_CAMERA_REQUEST_H__\n \n+#include <map>\n #include <memory>\n+#include <mutex>\n #include <vector>\n \n #include <libcamera/base/class.h>\n@@ -43,6 +45,10 @@ public:\n \t\tCamera3RequestDescriptor *request;\n \t};\n \n+\t/* Keeps track of streams requiring post-processing. */\n+\tstd::map<CameraStream *, StreamBuffer *> pendingStreamsToProcess_;\n+\tstd::mutex streamsProcessMutex_;\n+\n \tCamera3RequestDescriptor(libcamera::Camera *camera,\n \t\t\t\t const camera3_capture_request_t *camera3Request);\n \t~Camera3RequestDescriptor();\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex 282b19b0..dac216d6 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -22,6 +22,7 @@\n #include \"camera_capabilities.h\"\n #include \"camera_device.h\"\n #include \"camera_metadata.h\"\n+#include \"post_processor.h\"\n \n using namespace libcamera;\n \n@@ -97,6 +98,20 @@ int CameraStream::configure()\n \t\tint ret = postProcessor_->configure(configuration(), output);\n \t\tif (ret)\n \t\t\treturn ret;\n+\n+\t\tpostProcessor_->processComplete.connect(\n+\t\t\tthis, [&](Camera3RequestDescriptor::StreamBuffer *streamBuffer,\n+\t\t\t\t PostProcessor::Status status) {\n+\t\t\t\tCamera3RequestDescriptor::Status bufferStatus;\n+\n+\t\t\t\tif (status == PostProcessor::Status::Success)\n+\t\t\t\t\tbufferStatus = Camera3RequestDescriptor::Status::Success;\n+\t\t\t\telse\n+\t\t\t\t\tbufferStatus = Camera3RequestDescriptor::Status::Error;\n+\n+\t\t\t\tcameraDevice_->streamProcessingComplete(streamBuffer,\n+\t\t\t\t\t\t\t\t\tbufferStatus);\n+\t\t\t});\n \t}\n \n \tif (type_ == Type::Internal) {\ndiff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\nindex 240e29f6..5e8c61fc 100644\n--- a/src/android/jpeg/post_processor_jpeg.cpp\n+++ b/src/android/jpeg/post_processor_jpeg.cpp\n@@ -198,6 +198,7 @@ int PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBuf\n \t\t\t\t\t exif.data(), quality);\n \tif (jpeg_size < 0) {\n \t\tLOG(JPEG, Error) << \"Failed to encode stream image\";\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn jpeg_size;\n \t}\n \n@@ -211,6 +212,7 @@ int PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBuf\n \n \t/* Update the JPEG result Metadata. */\n \tresultMetadata->addEntry(ANDROID_JPEG_SIZE, jpeg_size);\n+\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Success);\n \n \treturn 0;\n }\ndiff --git a/src/android/post_processor.h b/src/android/post_processor.h\nindex 128161c8..4ac74fcf 100644\n--- a/src/android/post_processor.h\n+++ b/src/android/post_processor.h\n@@ -7,6 +7,8 @@\n #ifndef __ANDROID_POST_PROCESSOR_H__\n #define __ANDROID_POST_PROCESSOR_H__\n \n+#include <libcamera/base/signal.h>\n+\n #include <libcamera/framebuffer.h>\n #include <libcamera/stream.h>\n \n@@ -16,11 +18,18 @@\n class PostProcessor\n {\n public:\n+\tenum class Status {\n+\t\tError,\n+\t\tSuccess\n+\t};\n+\n \tvirtual ~PostProcessor() = default;\n \n \tvirtual int configure(const libcamera::StreamConfiguration &inCfg,\n \t\t\t const libcamera::StreamConfiguration &outCfg) = 0;\n \tvirtual int process(Camera3RequestDescriptor::StreamBuffer *streamBuffer) = 0;\n+\n+\tlibcamera::Signal<Camera3RequestDescriptor::StreamBuffer *, Status> processComplete;\n };\n \n #endif /* __ANDROID_POST_PROCESSOR_H__ */\ndiff --git a/src/android/yuv/post_processor_yuv.cpp b/src/android/yuv/post_processor_yuv.cpp\nindex 70385ab3..05c4f1cf 100644\n--- a/src/android/yuv/post_processor_yuv.cpp\n+++ b/src/android/yuv/post_processor_yuv.cpp\n@@ -54,12 +54,15 @@ int PostProcessorYuv::process(Camera3RequestDescriptor::StreamBuffer *streamBuff\n \tconst FrameBuffer &source = *streamBuffer->srcBuffer;\n \tCameraBuffer *destination = streamBuffer->dstBuffer.get();\n \n-\tif (!isValidBuffers(source, *destination))\n+\tif (!isValidBuffers(source, *destination)) {\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n+\t}\n \n \tconst MappedFrameBuffer sourceMapped(&source, MappedFrameBuffer::MapFlag::Read);\n \tif (!sourceMapped.isValid()) {\n \t\tLOG(YUV, Error) << \"Failed to mmap camera frame buffer\";\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n@@ -77,9 +80,12 @@ int PostProcessorYuv::process(Camera3RequestDescriptor::StreamBuffer *streamBuff\n \t\t\t\t libyuv::FilterMode::kFilterBilinear);\n \tif (ret) {\n \t\tLOG(YUV, Error) << \"Failed NV12 scaling: \" << ret;\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n+\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Success);\n+\n \treturn 0;\n }\n \n", "prefixes": [ "libcamera-devel", "v7", "5/7" ] }