Patch Detail
Show a patch.
GET /api/1.1/patches/14309/?format=api
{ "id": 14309, "url": "https://patchwork.libcamera.org/api/1.1/patches/14309/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14309/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211026072148.164831-8-umang.jain@ideasonboard.com>", "date": "2021-10-26T07:21:48", "name": "[libcamera-devel,v8,7/7] android: post_processor: Make post processing async", "commit_ref": "b1cefe38f360915ad597ab2934c009bd1e46d10d", "pull_url": null, "state": "accepted", "archived": false, "hash": "a024138980effdbbdcfd0ddca51f1f4f5ac8d919", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/14309/mbox/", "series": [ { "id": 2654, "url": "https://patchwork.libcamera.org/api/1.1/series/2654/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2654", "date": "2021-10-26T07:21:41", "name": "Async Post Processor", "version": 8, "mbox": "https://patchwork.libcamera.org/series/2654/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14309/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14309/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id AB2BCBF415\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 26 Oct 2021 07:22:11 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 709C164882;\n\tTue, 26 Oct 2021 09:22:11 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4C83164886\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 26 Oct 2021 09:22:09 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.211])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 2B2883F0;\n\tTue, 26 Oct 2021 09:22:08 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"v0PQ8nLZ\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1635232929;\n\tbh=I+y3A076cYL7/nTH2ebJdUvfwWWQcgULoFQMMaTdCQY=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=v0PQ8nLZZ8KwfVL1ZfpnM55CUuJW/Obvy/58wVLTDTeBsXMOljLlKSBowBy6DB/3R\n\trI5DmUtt98RGjudaPWhOMhCK33E0SY4XnFW6sVFuI2zuZfMKiOpiipqq8nQOdYh8jt\n\tmdkewAS+qMDA5v35CwIHORxUpUrtpz7BDmZgWQow=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Tue, 26 Oct 2021 12:51:48 +0530", "Message-Id": "<20211026072148.164831-8-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20211026072148.164831-1-umang.jain@ideasonboard.com>", "References": "<20211026072148.164831-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v8 7/7] android: post_processor: Make post\n\tprocessing async", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Introduce a dedicated worker class derived from libcamera::Thread.\nThe worker class maintains a queue for post-processing requests\nand waits for a post-processing request to become available.\nIt will process them as per FIFO before de-queuing it from the\nqueue.\n\nThe entire post-processing handling iteration is locked under\nstreamsProcessMutex_ which helps us to queue all the post-processing\nrequest at once, before any of the post-processing completion slot\n(streamProcessingComplete()) is allowed to run for post-processing\nrequests completing in parallel. This helps us to manage both\nsynchronous and asynchronous errors encountered during the entire\npost processing operation. Since a post-processing operation can\neven complete after CameraDevice::requestComplete() has returned,\nwe need to check and complete the descriptor from\nstreamProcessingComplete() running in the PostProcessorWorker's\nthread.\n\nThis patch also implements a flush() for the PostProcessorWorker\nclass which is responsible to purge post-processing requests\nqueued up while a camera is stopping/flushing. It is hooked with\nCameraStream::flush(), which isn't used currently but will be\nused when we handle flush/stop scenarios in greater detail\nsubsequently (in a different patchset).\n\nThe libcamera request completion handler CameraDevice::requestComplete()\nassumes that the request that has just completed is at the front of the\nqueue. Now that the post-processor runs asynchronously, this isn't true\nanymore, a request being post-processed will stay in the queue and a new\nlibcamera request may complete. Remove that assumption, and use the\nrequest cookie to obtain the Camera3RequestDescriptor.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\nSigned-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\nReviewed-by: Hirokazu Honda <hiroh@chromium.org>\n---\n src/android/camera_device.cpp | 48 +++++++---------\n src/android/camera_stream.cpp | 101 ++++++++++++++++++++++++++++++++--\n src/android/camera_stream.h | 38 ++++++++++++-\n 3 files changed, 153 insertions(+), 34 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex fed539f3..f2e0bdbd 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -1027,29 +1027,8 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \n void CameraDevice::requestComplete(Request *request)\n {\n-\tCamera3RequestDescriptor *descriptor;\n-\t{\n-\t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tASSERT(!descriptors_.empty());\n-\t\tdescriptor = descriptors_.front().get();\n-\t}\n-\n-\tif (descriptor->request_->cookie() != request->cookie()) {\n-\t\t/*\n-\t\t * \\todo Clarify if the Camera has to be closed on\n-\t\t * ERROR_DEVICE.\n-\t\t */\n-\t\tLOG(HAL, Error)\n-\t\t\t<< \"Out-of-order completion for request \"\n-\t\t\t<< utils::hex(request->cookie());\n-\n-\t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tdescriptors_.pop();\n-\n-\t\tnotifyError(0, nullptr, CAMERA3_MSG_ERROR_DEVICE);\n-\n-\t\treturn;\n-\t}\n+\tCamera3RequestDescriptor *descriptor =\n+\t\treinterpret_cast<Camera3RequestDescriptor *>(request->cookie());\n \n \t/*\n \t * Prepare the capture result for the Android camera stack.\n@@ -1124,9 +1103,13 @@ void CameraDevice::requestComplete(Request *request)\n \t}\n \n \t/* Handle post-processing. */\n+\tMutexLocker locker(descriptor->streamsProcessMutex_);\n+\n \t/*\n-\t * \\todo Protect the loop below with streamsProcessMutex_ when post\n-\t * processor runs asynchronously.\n+\t * Queue all the post-processing streams request at once. The completion\n+\t * slot streamProcessingComplete() can only execute when we are out\n+\t * this critical section. This helps to handle synchronous errors here\n+\t * itself.\n \t */\n \tauto iter = descriptor->pendingStreamsToProcess_.begin();\n \twhile (iter != descriptor->pendingStreamsToProcess_.end()) {\n@@ -1158,8 +1141,10 @@ void CameraDevice::requestComplete(Request *request)\n \t\t}\n \t}\n \n-\tif (descriptor->pendingStreamsToProcess_.empty())\n+\tif (descriptor->pendingStreamsToProcess_.empty()) {\n+\t\tlocker.unlock();\n \t\tcompleteDescriptor(descriptor);\n+\t}\n }\n \n void CameraDevice::completeDescriptor(Camera3RequestDescriptor *descriptor)\n@@ -1242,9 +1227,16 @@ void CameraDevice::streamProcessingComplete(Camera3RequestDescriptor::StreamBuff\n \t\tstreamBuffer->stream->putBuffer(streamBuffer->internalBuffer);\n \n \tCamera3RequestDescriptor *request = streamBuffer->request;\n-\tMutexLocker locker(request->streamsProcessMutex_);\n \n-\trequest->pendingStreamsToProcess_.erase(streamBuffer->stream);\n+\t{\n+\t\tMutexLocker locker(request->streamsProcessMutex_);\n+\n+\t\trequest->pendingStreamsToProcess_.erase(streamBuffer->stream);\n+\t\tif (!request->pendingStreamsToProcess_.empty())\n+\t\t\treturn;\n+\t}\n+\n+\tcompleteDescriptor(streamBuffer->request);\n }\n \n std::string CameraDevice::logPrefix() const\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex fed99022..9023c13c 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -99,6 +99,7 @@ int CameraStream::configure()\n \t\tif (ret)\n \t\t\treturn ret;\n \n+\t\tworker_ = std::make_unique<PostProcessorWorker>(postProcessor_.get());\n \t\tpostProcessor_->processComplete.connect(\n \t\t\tthis, [&](Camera3RequestDescriptor::StreamBuffer *streamBuffer,\n \t\t\t\t PostProcessor::Status status) {\n@@ -112,6 +113,8 @@ int CameraStream::configure()\n \t\t\t\tcameraDevice_->streamProcessingComplete(streamBuffer,\n \t\t\t\t\t\t\t\t\tbufferStatus);\n \t\t\t});\n+\n+\t\tworker_->start();\n \t}\n \n \tif (type_ == Type::Internal) {\n@@ -178,10 +181,6 @@ int CameraStream::process(Camera3RequestDescriptor::StreamBuffer *streamBuffer)\n \t\tstreamBuffer->fence = -1;\n \t}\n \n-\t/*\n-\t * \\todo Buffer mapping and processing should be moved to a\n-\t * separate thread.\n-\t */\n \tconst StreamConfiguration &output = configuration();\n \tstreamBuffer->dstBuffer = std::make_unique<CameraBuffer>(\n \t\t*streamBuffer->camera3Buffer, output.pixelFormat, output.size,\n@@ -191,11 +190,19 @@ int CameraStream::process(Camera3RequestDescriptor::StreamBuffer *streamBuffer)\n \t\treturn -EINVAL;\n \t}\n \n-\tpostProcessor_->process(streamBuffer);\n+\tworker_->queueRequest(streamBuffer);\n \n \treturn 0;\n }\n \n+void CameraStream::flush()\n+{\n+\tif (!postProcessor_)\n+\t\treturn;\n+\n+\tworker_->flush();\n+}\n+\n FrameBuffer *CameraStream::getBuffer()\n {\n \tif (!allocator_)\n@@ -223,3 +230,87 @@ void CameraStream::putBuffer(FrameBuffer *buffer)\n \n \tbuffers_.push_back(buffer);\n }\n+\n+CameraStream::PostProcessorWorker::PostProcessorWorker(PostProcessor *postProcessor)\n+\t: postProcessor_(postProcessor)\n+{\n+}\n+\n+CameraStream::PostProcessorWorker::~PostProcessorWorker()\n+{\n+\t{\n+\t\tlibcamera::MutexLocker lock(mutex_);\n+\t\tstate_ = State::Stopped;\n+\t}\n+\n+\tcv_.notify_one();\n+\twait();\n+}\n+\n+void CameraStream::PostProcessorWorker::start()\n+{\n+\t{\n+\t\tlibcamera::MutexLocker lock(mutex_);\n+\t\tASSERT(state_ != State::Running);\n+\t\tstate_ = State::Running;\n+\t}\n+\n+\tThread::start();\n+}\n+\n+void CameraStream::PostProcessorWorker::queueRequest(Camera3RequestDescriptor::StreamBuffer *dest)\n+{\n+\t{\n+\t\tMutexLocker lock(mutex_);\n+\t\tASSERT(state_ == State::Running);\n+\t\trequests_.push(dest);\n+\t}\n+\n+\tcv_.notify_one();\n+}\n+\n+void CameraStream::PostProcessorWorker::run()\n+{\n+\tMutexLocker locker(mutex_);\n+\n+\twhile (1) {\n+\t\tcv_.wait(locker, [&] {\n+\t\t\treturn state_ != State::Running || !requests_.empty();\n+\t\t});\n+\n+\t\tif (state_ != State::Running)\n+\t\t\tbreak;\n+\n+\t\tCamera3RequestDescriptor::StreamBuffer *streamBuffer = requests_.front();\n+\t\trequests_.pop();\n+\t\tlocker.unlock();\n+\n+\t\tpostProcessor_->process(streamBuffer);\n+\n+\t\tlocker.lock();\n+\t}\n+\n+\tif (state_ == State::Flushing) {\n+\t\tstd::queue<Camera3RequestDescriptor::StreamBuffer *> requests =\n+\t\t\tstd::move(requests_);\n+\t\tlocker.unlock();\n+\n+\t\twhile (!requests.empty()) {\n+\t\t\tpostProcessor_->processComplete.emit(\n+\t\t\t\trequests.front(), PostProcessor::Status::Error);\n+\t\t\trequests.pop();\n+\t\t}\n+\n+\t\tlocker.lock();\n+\t\tstate_ = State::Stopped;\n+\t}\n+}\n+\n+void CameraStream::PostProcessorWorker::flush()\n+{\n+\tlibcamera::MutexLocker lock(mutex_);\n+\tstate_ = State::Flushing;\n+\tlock.unlock();\n+\n+\tcv_.notify_one();\n+}\ndiff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\nindex e74a9a3b..0c402deb 100644\n--- a/src/android/camera_stream.h\n+++ b/src/android/camera_stream.h\n@@ -7,12 +7,16 @@\n #ifndef __ANDROID_CAMERA_STREAM_H__\n #define __ANDROID_CAMERA_STREAM_H__\n \n+#include <condition_variable>\n #include <memory>\n #include <mutex>\n+#include <queue>\n #include <vector>\n \n #include <hardware/camera3.h>\n \n+#include <libcamera/base/thread.h>\n+\n #include <libcamera/camera.h>\n #include <libcamera/framebuffer.h>\n #include <libcamera/framebuffer_allocator.h>\n@@ -20,9 +24,9 @@\n #include <libcamera/pixel_format.h>\n \n #include \"camera_request.h\"\n+#include \"post_processor.h\"\n \n class CameraDevice;\n-class PostProcessor;\n \n class CameraStream\n {\n@@ -124,8 +128,38 @@ public:\n \tint process(Camera3RequestDescriptor::StreamBuffer *streamBuffer);\n \tlibcamera::FrameBuffer *getBuffer();\n \tvoid putBuffer(libcamera::FrameBuffer *buffer);\n+\tvoid flush();\n \n private:\n+\tclass PostProcessorWorker : public libcamera::Thread\n+\t{\n+\tpublic:\n+\t\tenum class State {\n+\t\t\tStopped,\n+\t\t\tRunning,\n+\t\t\tFlushing,\n+\t\t};\n+\n+\t\tPostProcessorWorker(PostProcessor *postProcessor);\n+\t\t~PostProcessorWorker();\n+\n+\t\tvoid start();\n+\t\tvoid queueRequest(Camera3RequestDescriptor::StreamBuffer *request);\n+\t\tvoid flush();\n+\n+\tprotected:\n+\t\tvoid run() override;\n+\n+\tprivate:\n+\t\tPostProcessor *postProcessor_;\n+\n+\t\tlibcamera::Mutex mutex_;\n+\t\tstd::condition_variable cv_;\n+\n+\t\tstd::queue<Camera3RequestDescriptor::StreamBuffer *> requests_;\n+\t\tState state_ = State::Stopped;\n+\t};\n+\n \tint waitFence(int fence);\n \n \tCameraDevice *const cameraDevice_;\n@@ -142,6 +176,8 @@ private:\n \t */\n \tstd::unique_ptr<std::mutex> mutex_;\n \tstd::unique_ptr<PostProcessor> postProcessor_;\n+\n+\tstd::unique_ptr<PostProcessorWorker> worker_;\n };\n \n #endif /* __ANDROID_CAMERA_STREAM__ */\n", "prefixes": [ "libcamera-devel", "v8", "7/7" ] }