Patch Detail
Show a patch.
GET /api/1.1/patches/14291/?format=api
{ "id": 14291, "url": "https://patchwork.libcamera.org/api/1.1/patches/14291/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14291/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211023103302.152769-6-umang.jain@ideasonboard.com>", "date": "2021-10-23T10:33:00", "name": "[libcamera-devel,v6,5/7] android: Track and notify post processing of streams", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "6af737dfea89f8a86ca1a21b91e4051209436bdb", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/14291/mbox/", "series": [ { "id": 2652, "url": "https://patchwork.libcamera.org/api/1.1/series/2652/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2652", "date": "2021-10-23T10:32:55", "name": "Async Post Processor", "version": 6, "mbox": "https://patchwork.libcamera.org/series/2652/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14291/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14291/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 82BC6BDB1C\n\tfor <parsemail@patchwork.libcamera.org>;\n\tSat, 23 Oct 2021 10:33:21 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3B1B464872;\n\tSat, 23 Oct 2021 12:33:21 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B50D764870\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tSat, 23 Oct 2021 12:33:19 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.213])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8AE73547;\n\tSat, 23 Oct 2021 12:33:18 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"N95KbetL\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1634985199;\n\tbh=fUoLBab6iCL7Atuu2/jNcWYcAX7Nwx1kRd8tGN6qetI=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=N95KbetL8u6rg4MvtXMIken41/dJuDdukSUCNfaDK9Ki7iWFhFsA70t4zs7PLJ/nf\n\tDr8eGZs7UJT5lVy9wQe4DgyzGcad0rKSLnRjT98fqDkdtGjTaiEFV7lm8GR2k5LBQJ\n\tjRN/DjKpWWz0H0c9lQO5LpmKlXGH+hnwJuREQWmU=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Sat, 23 Oct 2021 16:03:00 +0530", "Message-Id": "<20211023103302.152769-6-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20211023103302.152769-1-umang.jain@ideasonboard.com>", "References": "<20211023103302.152769-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v6 5/7] android: Track and notify post\n\tprocessing of streams", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Notify that the post processing for a request has been completed,\nvia a signal. The signal emit with a context pointer along with status\nof the buffer. The function CameraDevice::streamProcessingComplete() will\nfinally set the status on the request descriptor and complete the\ndescriptor if all the streams requiring post processing are completed.\nIf buffer status obtained is in error state, notify the status to the\nframework and set the overall error status on the descriptor via\nsetBufferStatus().\n\nWe need to track the number of streams requiring post-processing\nper Camera3RequestDescriptor (i.e. per capture request). Introduce\na std::map<> to track the post-processing of streams. The nodes\nare dropped from the map when a particular stream post processing\nis completed (or on error paths). A std::map is selected for tracking\npost-processing requests, since we will move post-processing to be\nasynchronous in subsequent commits. A vector or queue will not be\nsuitable then as the sequential order of post-processing completion\nof various requests won't be guaranteed then.\n\nA streamProcessMutex_ has been introduced here as well, which will be\napplicable to guard access to descriptor's pendingStreamsToProcess_ when\npost-processing is moved to be asynchronous in subsequent commits.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/android/camera_device.cpp | 95 ++++++++++++++++++------\n src/android/camera_device.h | 4 +\n src/android/camera_request.h | 6 ++\n src/android/camera_stream.cpp | 15 ++++\n src/android/jpeg/post_processor_jpeg.cpp | 2 +\n src/android/post_processor.h | 9 +++\n src/android/yuv/post_processor_yuv.cpp | 8 +-\n 7 files changed, 114 insertions(+), 25 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex 2a98a2e6..3114def0 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -926,6 +926,8 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\t * Request.\n \t\t\t */\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (mapped)\";\n+\t\t\tdescriptor->pendingStreamsToProcess_.insert(\n+\t\t\t\t{ cameraStream, &buffer });\n \t\t\tcontinue;\n \n \t\tcase CameraStream::Type::Direct:\n@@ -955,6 +957,9 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\tframeBuffer = cameraStream->getBuffer();\n \t\t\tbuffer.internalBuffer = frameBuffer;\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (internal)\";\n+\n+\t\t\tdescriptor->pendingStreamsToProcess_.insert(\n+\t\t\t\t{ cameraStream, &buffer });\n \t\t\tbreak;\n \t\t}\n \n@@ -1118,43 +1123,46 @@ void CameraDevice::requestComplete(Request *request)\n \t}\n \n \t/* Handle post-processing. */\n-\tbool hasPostProcessingErrors = false;\n-\tfor (auto &buffer : descriptor->buffers_) {\n-\t\tCameraStream *stream = buffer.stream;\n-\n-\t\tif (stream->type() == CameraStream::Type::Direct)\n-\t\t\tcontinue;\n+\tbool needPostProcessing = false;\n+\t/*\n+\t * \\todo Protect the loop below with streamProcessMutex_ when post\n+\t * processor runs asynchronously.\n+\t */\n+\tauto iter = descriptor->pendingStreamsToProcess_.begin();\n+\twhile (descriptor->pendingStreamsToProcess_.size() > 0) {\n+\t\tCameraStream *stream = iter->first;\n+\t\tCamera3RequestDescriptor::StreamBuffer *buffer = iter->second;\n+\t\tneedPostProcessing = true;\n \n \t\tFrameBuffer *src = request->findBuffer(stream->stream());\n \t\tif (!src) {\n \t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n-\t\t\tbuffer.status = Camera3RequestDescriptor::Status::Error;\n-\t\t\tnotifyError(descriptor->frameNumber_, stream->camera3Stream(),\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n-\t\t\thasPostProcessingErrors = true;\n+\t\t\tsetBufferStatus(*buffer, Camera3RequestDescriptor::Status::Error);\n+\t\t\titer = descriptor->pendingStreamsToProcess_.erase(iter);\n \t\t\tcontinue;\n \t\t}\n \n-\t\tint ret = stream->process(*src, buffer);\n+\t\t++iter;\n+\t\tint ret = stream->process(*src, *buffer);\n+\t\tif (ret) {\n+\t\t\tsetBufferStatus(*buffer, Camera3RequestDescriptor::Status::Error);\n+\t\t\tdescriptor->pendingStreamsToProcess_.erase(stream);\n+\t\t}\n+\t}\n \n+\tif (needPostProcessing) {\n \t\t/*\n-\t\t * If the framebuffer is internal to CameraStream return it back\n-\t\t * now that we're done processing it.\n+\t\t * \\todo We will require to check if we failed to queue\n+\t\t * post-processing requests when we migrate to post-processor\n+\t\t * running asynchronously.\n+\t\t *\n+\t\t * if (descriptor->pendingStreamsToProcess_.size() == 0)\n+\t\t *\tcompleteDescriptor(descriptor);\n \t\t */\n-\t\tif (buffer.internalBuffer)\n-\t\t\tstream->putBuffer(buffer.internalBuffer);\n \n-\t\tif (ret) {\n-\t\t\tbuffer.status = Camera3RequestDescriptor::Status::Error;\n-\t\t\thasPostProcessingErrors = true;\n-\t\t\tnotifyError(descriptor->frameNumber_, stream->camera3Stream(),\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n-\t\t}\n+\t\treturn;\n \t}\n \n-\tif (hasPostProcessingErrors)\n-\t\tdescriptor->status_ = Camera3RequestDescriptor::Status::Error;\n-\n \tcompleteDescriptor(descriptor);\n }\n \n@@ -1210,6 +1218,45 @@ void CameraDevice::sendCaptureResults()\n \t}\n }\n \n+void CameraDevice::setBufferStatus(Camera3RequestDescriptor::StreamBuffer &streamBuffer,\n+\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\t/*\n+\t * If the framebuffer is internal to CameraStream return it back now\n+\t * that we're done processing it.\n+\t */\n+\tif (streamBuffer.internalBuffer)\n+\t\tstreamBuffer.stream->putBuffer(streamBuffer.internalBuffer);\n+\n+\tstreamBuffer.status = status;\n+\tif (status != Camera3RequestDescriptor::Status::Success) {\n+\t\tnotifyError(streamBuffer.request->frameNumber_,\n+\t\t\t streamBuffer.stream->camera3Stream(),\n+\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\n+\t\t/* Also set error status on entire request descriptor. */\n+\t\tstreamBuffer.request->status_ =\n+\t\t\tCamera3RequestDescriptor::Status::Error;\n+\t}\n+}\n+\n+void CameraDevice::streamProcessingComplete(Camera3RequestDescriptor::StreamBuffer *streamBuffer,\n+\t\t\t\t\t Camera3RequestDescriptor::Status status)\n+{\n+\tCamera3RequestDescriptor *request = streamBuffer->request;\n+\tMutexLocker locker(request->streamsProcessMutex_);\n+\n+\tsetBufferStatus(*streamBuffer, status);\n+\trequest->pendingStreamsToProcess_.erase(streamBuffer->stream);\n+\n+\tif (request->pendingStreamsToProcess_.size() > 0)\n+\t\treturn;\n+\n+\tlocker.unlock();\n+\n+\tcompleteDescriptor(streamBuffer->request);\n+}\n+\n std::string CameraDevice::logPrefix() const\n {\n \treturn \"'\" + camera_->id() + \"'\";\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex e544f2bd..2a414020 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -66,6 +66,8 @@ public:\n \tint configureStreams(camera3_stream_configuration_t *stream_list);\n \tint processCaptureRequest(camera3_capture_request_t *request);\n \tvoid requestComplete(libcamera::Request *request);\n+\tvoid streamProcessingComplete(Camera3RequestDescriptor::StreamBuffer *bufferStream,\n+\t\t\t\t Camera3RequestDescriptor::Status status);\n \n protected:\n \tstd::string logPrefix() const override;\n@@ -95,6 +97,8 @@ private:\n \tint processControls(Camera3RequestDescriptor *descriptor);\n \tvoid completeDescriptor(Camera3RequestDescriptor *descriptor);\n \tvoid sendCaptureResults();\n+\tvoid setBufferStatus(Camera3RequestDescriptor::StreamBuffer &buffer,\n+\t\t\t Camera3RequestDescriptor::Status status);\n \tstd::unique_ptr<CameraMetadata> getResultMetadata(\n \t\tconst Camera3RequestDescriptor &descriptor) const;\n \ndiff --git a/src/android/camera_request.h b/src/android/camera_request.h\nindex c4bc5d6e..cc2b7035 100644\n--- a/src/android/camera_request.h\n+++ b/src/android/camera_request.h\n@@ -7,7 +7,9 @@\n #ifndef __ANDROID_CAMERA_REQUEST_H__\n #define __ANDROID_CAMERA_REQUEST_H__\n \n+#include <map>\n #include <memory>\n+#include <mutex>\n #include <vector>\n \n #include <libcamera/base/class.h>\n@@ -43,6 +45,10 @@ public:\n \t\tCamera3RequestDescriptor *request;\n \t};\n \n+\t/* Keeps track of streams requiring post-processing. */\n+\tstd::map<CameraStream *, StreamBuffer *> pendingStreamsToProcess_;\n+\tstd::mutex streamsProcessMutex_;\n+\n \tCamera3RequestDescriptor(libcamera::Camera *camera,\n \t\t\t\t const camera3_capture_request_t *camera3Request);\n \t~Camera3RequestDescriptor();\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex 0e268cdf..4e275cde 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -22,6 +22,7 @@\n #include \"camera_capabilities.h\"\n #include \"camera_device.h\"\n #include \"camera_metadata.h\"\n+#include \"post_processor.h\"\n \n using namespace libcamera;\n \n@@ -97,6 +98,20 @@ int CameraStream::configure()\n \t\tint ret = postProcessor_->configure(configuration(), output);\n \t\tif (ret)\n \t\t\treturn ret;\n+\n+\t\tpostProcessor_->processComplete.connect(\n+\t\t\tthis, [&](Camera3RequestDescriptor::StreamBuffer *streamBuffer,\n+\t\t\t\t PostProcessor::Status status) {\n+\t\t\t\tCamera3RequestDescriptor::Status bufferStatus;\n+\n+\t\t\t\tif (status == PostProcessor::Status::Success)\n+\t\t\t\t\tbufferStatus = Camera3RequestDescriptor::Status::Success;\n+\t\t\t\telse\n+\t\t\t\t\tbufferStatus = Camera3RequestDescriptor::Status::Error;\n+\n+\t\t\t\tcameraDevice_->streamProcessingComplete(streamBuffer,\n+\t\t\t\t\t\t\t\t\tbufferStatus);\n+\t\t\t});\n \t}\n \n \tif (type_ == Type::Internal) {\ndiff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\nindex da71f113..cbbe7128 100644\n--- a/src/android/jpeg/post_processor_jpeg.cpp\n+++ b/src/android/jpeg/post_processor_jpeg.cpp\n@@ -198,6 +198,7 @@ int PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBuf\n \t\t\t\t\t exif.data(), quality);\n \tif (jpeg_size < 0) {\n \t\tLOG(JPEG, Error) << \"Failed to encode stream image\";\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn jpeg_size;\n \t}\n \n@@ -211,6 +212,7 @@ int PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBuf\n \n \t/* Update the JPEG result Metadata. */\n \tresultMetadata->addEntry(ANDROID_JPEG_SIZE, jpeg_size);\n+\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Success);\n \n \treturn 0;\n }\ndiff --git a/src/android/post_processor.h b/src/android/post_processor.h\nindex 128161c8..4ac74fcf 100644\n--- a/src/android/post_processor.h\n+++ b/src/android/post_processor.h\n@@ -7,6 +7,8 @@\n #ifndef __ANDROID_POST_PROCESSOR_H__\n #define __ANDROID_POST_PROCESSOR_H__\n \n+#include <libcamera/base/signal.h>\n+\n #include <libcamera/framebuffer.h>\n #include <libcamera/stream.h>\n \n@@ -16,11 +18,18 @@\n class PostProcessor\n {\n public:\n+\tenum class Status {\n+\t\tError,\n+\t\tSuccess\n+\t};\n+\n \tvirtual ~PostProcessor() = default;\n \n \tvirtual int configure(const libcamera::StreamConfiguration &inCfg,\n \t\t\t const libcamera::StreamConfiguration &outCfg) = 0;\n \tvirtual int process(Camera3RequestDescriptor::StreamBuffer *streamBuffer) = 0;\n+\n+\tlibcamera::Signal<Camera3RequestDescriptor::StreamBuffer *, Status> processComplete;\n };\n \n #endif /* __ANDROID_POST_PROCESSOR_H__ */\ndiff --git a/src/android/yuv/post_processor_yuv.cpp b/src/android/yuv/post_processor_yuv.cpp\nindex eeb8f1f4..8e77bf57 100644\n--- a/src/android/yuv/post_processor_yuv.cpp\n+++ b/src/android/yuv/post_processor_yuv.cpp\n@@ -54,12 +54,15 @@ int PostProcessorYuv::process(Camera3RequestDescriptor::StreamBuffer *streamBuff\n \tconst FrameBuffer &source = *streamBuffer->srcBuffer;\n \tCameraBuffer *destination = streamBuffer->destBuffer.get();\n \n-\tif (!isValidBuffers(source, *destination))\n+\tif (!isValidBuffers(source, *destination)) {\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n+\t}\n \n \tconst MappedFrameBuffer sourceMapped(&source, MappedFrameBuffer::MapFlag::Read);\n \tif (!sourceMapped.isValid()) {\n \t\tLOG(YUV, Error) << \"Failed to mmap camera frame buffer\";\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n@@ -77,9 +80,12 @@ int PostProcessorYuv::process(Camera3RequestDescriptor::StreamBuffer *streamBuff\n \t\t\t\t libyuv::FilterMode::kFilterBilinear);\n \tif (ret) {\n \t\tLOG(YUV, Error) << \"Failed NV12 scaling: \" << ret;\n+\t\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Error);\n \t\treturn -EINVAL;\n \t}\n \n+\tprocessComplete.emit(streamBuffer, PostProcessor::Status::Success);\n+\n \treturn 0;\n }\n \n", "prefixes": [ "libcamera-devel", "v6", "5/7" ] }