Patch Detail
Show a patch.
GET /api/patches/13806/?format=api
{ "id": 13806, "url": "https://patchwork.libcamera.org/api/patches/13806/?format=api", "web_url": "https://patchwork.libcamera.org/patch/13806/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210910070638.467294-9-umang.jain@ideasonboard.com>", "date": "2021-09-10T07:06:37", "name": "[libcamera-devel,v2,8/9] android: camera_stream: Run post-processor in a separate thread", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "29cad040dd720c8923da778e62eda00b9cefe94c", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": { "id": 12, "url": "https://patchwork.libcamera.org/api/users/12/?format=api", "username": "uajain", "first_name": "Umang", "last_name": "Jain", "email": "umang.jain@ideasonboard.com" }, "mbox": "https://patchwork.libcamera.org/patch/13806/mbox/", "series": [ { "id": 2513, "url": "https://patchwork.libcamera.org/api/series/2513/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2513", "date": "2021-09-10T07:06:29", "name": "Async Post Processor", "version": 2, "mbox": "https://patchwork.libcamera.org/series/2513/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/13806/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/13806/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 1E01FBDC71\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 10 Sep 2021 07:07:05 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B84126917B;\n\tFri, 10 Sep 2021 09:07:04 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9700B6916E\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 10 Sep 2021 09:07:02 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.149])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8DB4B883;\n\tFri, 10 Sep 2021 09:07:01 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"Js9BbFM9\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1631257622;\n\tbh=GhFCseGpcp3egLXZPZSTCi008bvqr/gET9XO/EDdE6c=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=Js9BbFM9nHF3wCdXq8Ssm5eGBmvtxPNNLA0CfNNHnsPIfsE9R93fvc0OYRPUwBC/U\n\t+v6omaGw14mwKTiNw0nKs/Y6OXhmFQdEc+7ni/T7buN5w4c0dVOqvDKM2WUBEsreQJ\n\thEXw+7FND291MHWPnf2XR22p8H5QNsf2shsQu4QI=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 10 Sep 2021 12:36:37 +0530", "Message-Id": "<20210910070638.467294-9-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20210910070638.467294-1-umang.jain@ideasonboard.com>", "References": "<20210910070638.467294-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v2 8/9] android: camera_stream: Run\n\tpost-processor in a separate thread", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "In CameraStream, introduce a new private PostProcessorWorker\nclass derived from Object. A instance of PostProcessorWorker\nis moved to a separate thread instance which will be responsible\nto run the post-processor.\n\nRunning PostProcessor asynchronously should entail that all the\ndata context needed by the PostProcessor should remain valid for\nthe entire duration of its run. Most of the context preserving\npart has been addressed in the previous commits, we just need to\nensure the source framebuffer data that comes via Camera::Request,\nshould remain valid for the entire duration of post-processing\nrunning asynchronously. In order to so, we maintain a separate\ncopy of the framebuffer data and add it to the Camera3RequestDescriptor\nstructure in which we preserve rest of the context.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/android/camera_device.cpp | 19 ++++++++++++++++++-\n src/android/camera_device.h | 4 ++++\n src/android/camera_stream.cpp | 16 ++++++++++++++--\n src/android/camera_stream.h | 28 ++++++++++++++++++++++++++++\n 4 files changed, 64 insertions(+), 3 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex 988d4232..73eb5758 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -1174,13 +1174,29 @@ void CameraDevice::requestComplete(Request *request)\n \t\t\treturn;\n \t\t}\n \n+\t\tFrameBuffer *source = src;\n+\t\tif (cameraStream->type() != CameraStream::Type::Internal) {\n+\t\t\t/*\n+\t\t\t * The source buffer is owned by Request object which\n+\t\t\t * can be reused by libcamera. Since post-processor will\n+\t\t\t * run asynchrnously, we need to copy the request's\n+\t\t\t * frame buffer and use that as the source buffer for\n+\t\t\t * post processing.\n+\t\t\t */\n+\t\t\tfor (const auto &plane : src->planes())\n+\t\t\t\tdescriptor.srcPlanes_.push_back(plane);\n+\t\t\tdescriptor.srcFramebuffer_ =\n+\t\t\t\tstd::make_unique<FrameBuffer>(descriptor.srcPlanes_);\n+\t\t\tsource = descriptor.srcFramebuffer_.get();\n+\t\t}\n+\n \t\tstd::unique_ptr<Camera3RequestDescriptor> reqDescriptor =\n \t\t\tstd::make_unique<Camera3RequestDescriptor>();\n \t\t*reqDescriptor = std::move(descriptor);\n \t\tqueuedDescriptor_.push_back(std::move(reqDescriptor));\n \n \t\tCamera3RequestDescriptor *currentDescriptor = queuedDescriptor_.back().get();\n-\t\tint ret = cameraStream->process(src,\n+\t\tint ret = cameraStream->process(source,\n \t\t\t\t\t\tcurrentDescriptor->destBuffer_.get(),\n \t\t\t\t\t\tcurrentDescriptor->settings_,\n \t\t\t\t\t\tcurrentDescriptor->resultMetadata_.get(),\n@@ -1255,6 +1271,7 @@ void CameraDevice::streamProcessingComplete(CameraStream *cameraStream,\n \n void CameraDevice::sendQueuedCaptureResults()\n {\n+\tMutexLocker lock(queuedDescriptorsMutex_);\n \twhile (!queuedDescriptor_.empty()) {\n \t\tstd::unique_ptr<Camera3RequestDescriptor> &d = queuedDescriptor_.front();\n \t\tif (d->status_ == Camera3RequestDescriptor::Pending)\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex b62d373c..ecdda06e 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -62,6 +62,9 @@ struct Camera3RequestDescriptor {\n \tcamera3_capture_result_t captureResult_;\n \tlibcamera::FrameBuffer *internalBuffer_;\n \tcompletionStatus status_;\n+\n+\tstd::unique_ptr<libcamera::FrameBuffer> srcFramebuffer_;\n+\tstd::vector<libcamera::FrameBuffer::Plane> srcPlanes_;\n };\n \n class CameraDevice : protected libcamera::Loggable\n@@ -147,6 +150,7 @@ private:\n \tlibcamera::Mutex descriptorsMutex_; /* Protects descriptors_. */\n \tstd::map<uint64_t, Camera3RequestDescriptor> descriptors_;\n \n+\tlibcamera::Mutex queuedDescriptorsMutex_; /* Protects queuedDescriptor_. */\n \tstd::deque<std::unique_ptr<Camera3RequestDescriptor>> queuedDescriptor_;\n \n \tstd::string maker_;\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex 5fd04bbf..845e2462 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -55,6 +55,15 @@ CameraStream::CameraStream(CameraDevice *const cameraDevice,\n \t\t * is what we instantiate here.\n \t\t */\n \t\tpostProcessor_ = std::make_unique<PostProcessorJpeg>(cameraDevice_);\n+\t\tppWorker_ = std::make_unique<PostProcessorWorker>(postProcessor_.get());\n+\n+\t\tthread_ = std::make_unique<libcamera::Thread>();\n+\t\tppWorker_->moveToThread(thread_.get());\n+\t\t/*\n+\t\t * \\todo: Class is MoveConstructible, so where to stop thread\n+\t\t * if we don't user-defined destructor? See RFC patch at the end.\n+\t\t */\n+\t\tthread_->start();\n \t}\n \n \tif (type == Type::Internal) {\n@@ -110,8 +119,11 @@ int CameraStream::process(const FrameBuffer *source,\n \tif (!postProcessor_)\n \t\treturn 0;\n \n-\treturn postProcessor_->process(source, destBuffer, requestMetadata,\n-\t\t\t\t resultMetadata, context);\n+\tppWorker_->invokeMethod(&PostProcessorWorker::process,\n+\t\t\t\tConnectionTypeQueued, source, destBuffer,\n+\t\t\t\trequestMetadata, resultMetadata, context);\n+\n+\treturn 0;\n }\n \n void CameraStream::postProcessingComplete(PostProcessor::Status status,\ndiff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\nindex 8097ddbc..dbb7eee3 100644\n--- a/src/android/camera_stream.h\n+++ b/src/android/camera_stream.h\n@@ -13,7 +13,9 @@\n \n #include <hardware/camera3.h>\n \n+#include <libcamera/base/object.h>\n #include <libcamera/base/signal.h>\n+#include <libcamera/base/thread.h>\n \n #include <libcamera/camera.h>\n #include <libcamera/framebuffer.h>\n@@ -23,6 +25,7 @@\n \n #include \"post_processor.h\"\n \n+class CameraBuffer;\n class CameraDevice;\n class CameraMetadata;\n \n@@ -137,6 +140,29 @@ public:\n \t};\n \n private:\n+\tclass PostProcessorWorker : public libcamera::Object\n+\t{\n+\tpublic:\n+\t\tPostProcessorWorker(PostProcessor *postProcessor)\n+\t\t{\n+\t\t\tpostProcessor_ = postProcessor;\n+\t\t}\n+\n+\t\tvoid process(const libcamera::FrameBuffer *source,\n+\t\t\t CameraBuffer *destination,\n+\t\t\t const CameraMetadata &requestMetadata,\n+\t\t\t CameraMetadata *resultMetadata,\n+\t\t\t const Camera3RequestDescriptor *context)\n+\t\t{\n+\t\t\tpostProcessor_->process(source, destination,\n+\t\t\t\t\t\trequestMetadata, resultMetadata,\n+\t\t\t\t\t\tcontext);\n+\t\t}\n+\n+\tprivate:\n+\t\tPostProcessor *postProcessor_;\n+\t};\n+\n \tvoid postProcessingComplete(PostProcessor::Status status,\n \t\t\t\t const Camera3RequestDescriptor *context);\n \n@@ -154,6 +180,8 @@ private:\n \t */\n \tstd::unique_ptr<std::mutex> mutex_;\n \tstd::unique_ptr<PostProcessor> postProcessor_;\n+\tstd::unique_ptr<PostProcessorWorker> ppWorker_;\n+\tstd::unique_ptr<libcamera::Thread> thread_;\n };\n \n #endif /* __ANDROID_CAMERA_STREAM__ */\n", "prefixes": [ "libcamera-devel", "v2", "8/9" ] }