Patch Detail
Show a patch.
GET /api/1.1/patches/13804/?format=api
{ "id": 13804, "url": "https://patchwork.libcamera.org/api/1.1/patches/13804/?format=api", "web_url": "https://patchwork.libcamera.org/patch/13804/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210910070638.467294-7-umang.jain@ideasonboard.com>", "date": "2021-09-10T07:06:35", "name": "[libcamera-devel,v2,6/9] android: camera_device: Add a queue for sending capture results", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "adbb0df5e4723da3c2960240ca0ff02ce45b241d", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": { "id": 12, "url": "https://patchwork.libcamera.org/api/1.1/users/12/?format=api", "username": "uajain", "first_name": "Umang", "last_name": "Jain", "email": "umang.jain@ideasonboard.com" }, "mbox": "https://patchwork.libcamera.org/patch/13804/mbox/", "series": [ { "id": 2513, "url": "https://patchwork.libcamera.org/api/1.1/series/2513/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2513", "date": "2021-09-10T07:06:29", "name": "Async Post Processor", "version": 2, "mbox": "https://patchwork.libcamera.org/series/2513/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/13804/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/13804/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 53416BDC71\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 10 Sep 2021 07:07:02 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 248BE69177;\n\tFri, 10 Sep 2021 09:07:02 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 90BD069169\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 10 Sep 2021 09:06:58 +0200 (CEST)", "from perceval.ideasonboard.com (unknown [103.251.226.149])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8F6F6BC0;\n\tFri, 10 Sep 2021 09:06:57 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"coRZ+SeZ\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1631257618;\n\tbh=OqW2lZGCcnuofLg+JxPbMd736XyeEBFLOPyj2TNi1wA=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=coRZ+SeZyQrRcxXa726pBAyCt3dyoj728WCg2o/jjCGsnjy5G6o6Ctmouj/CDGp6M\n\tdVMlL3qS2RH6amAiUX36bxCRfEe5vZQNoedIpr7AKWF4X51PTFFsQi1ZTm5Q5iZpg+\n\tp1XWVqNLUkDFm6y/uUeqbEiid6fFglWLSNsEZqSk=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 10 Sep 2021 12:36:35 +0530", "Message-Id": "<20210910070638.467294-7-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.31.1", "In-Reply-To": "<20210910070638.467294-1-umang.jain@ideasonboard.com>", "References": "<20210910070638.467294-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v2 6/9] android: camera_device: Add a\n\tqueue for sending capture results", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "When a camera capture request completes, the next step is to send the\ncapture results to the framework via process_capture_results(). All\nthe capture associated result metadata is prepared and populated. If\nany post-processing is required, it will also take place in the same\nthread (which can be blocking for a subtaintial amount of time) before\nthe results can be sent back to the framework.\n\nSubsequent patches will move the post-processing to run in a separate\nthread. In order to do so, there is few bits of groundwork which this\npatch entails. Mainly, we need to preserve the order of sending\nthe capture results back in which they were queued by the framework\nto the HAL (i.e. the order is sequential). Hence, we need to add a queue\nin which capture results can be queued with context.\n\nAs per this patch, the post-processor still runs synchronously as\nbefore, but it will queue up the current capture results with context.\nLater down the line, the capture results are dequeud in order and\nsent back to the framework via sendQueuedCaptureResults(). If the queue\nis empty or unused, the current behaviour is to skip the queue and\nsend capture results directly.\n\nThe context is preserved using Camera3RequestDescriptor utility\nstructure. It has been expanded accordingly to address the needs of\nthe functionality.\n---\nCheck if we can drop unique_ptr to camera3descriptor\n---\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/android/camera_device.cpp | 106 ++++++++++++++++++++++++++++++----\n src/android/camera_device.h | 22 +++++++\n src/android/camera_stream.cpp | 2 +\n 3 files changed, 119 insertions(+), 11 deletions(-)", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex 84549d04..7f04d225 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -240,6 +240,8 @@ Camera3RequestDescriptor::Camera3RequestDescriptor(\n \t/* Clone the controls associated with the camera3 request. */\n \tsettings_ = CameraMetadata(camera3Request->settings);\n \n+\tinternalBuffer_ = nullptr;\n+\n \t/*\n \t * Create the CaptureRequest, stored as a unique_ptr<> to tie its\n \t * lifetime to the descriptor.\n@@ -989,6 +991,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\t * once it has been processed.\n \t\t\t */\n \t\t\tbuffer = cameraStream->getBuffer();\n+\t\t\tdescriptor.internalBuffer_ = buffer;\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (internal)\";\n \t\t\tbreak;\n \t\t}\n@@ -1148,26 +1151,107 @@ void CameraDevice::requestComplete(Request *request)\n \t\t\tcontinue;\n \t\t}\n \n+\n+\t\t/*\n+\t\t * Save the current context of capture result and queue the\n+\t\t * descriptor before processing the camera stream.\n+\t\t *\n+\t\t * When the processing completes, the descriptor will be\n+\t\t * dequeued and capture results will be sent to the framework.\n+\t\t */\n+\t\tdescriptor.status_ = Camera3RequestDescriptor::Pending;\n+\t\tdescriptor.resultMetadata_ = std::move(resultMetadata);\n+\t\tdescriptor.captureResult_ = captureResult;\n+\n+\t\tstd::unique_ptr<Camera3RequestDescriptor> reqDescriptor =\n+\t\t\tstd::make_unique<Camera3RequestDescriptor>();\n+\t\t*reqDescriptor = std::move(descriptor);\n+\t\tqueuedDescriptor_.push_back(std::move(reqDescriptor));\n+\n+\t\tCamera3RequestDescriptor *currentDescriptor = queuedDescriptor_.back().get();\n \t\tint ret = cameraStream->process(src, *buffer.buffer,\n-\t\t\t\t\t\tdescriptor.settings_,\n-\t\t\t\t\t\tresultMetadata.get(),\n-\t\t\t\t\t\t&descriptor);\n+\t\t\t\t\t\tcurrentDescriptor->settings_,\n+\t\t\t\t\t\tcurrentDescriptor->resultMetadata_.get(),\n+\t\t\t\t\t\tcurrentDescriptor);\n+\t\treturn;\n+\t}\n+\n+\tif (queuedDescriptor_.empty()) {\n+\t\tcaptureResult.result = resultMetadata->get();\n+\t\tcallbacks_->process_capture_result(callbacks_, &captureResult);\n+\t} else {\n+\t\t/*\n+\t\t * Previous capture results waiting to be sent to framework\n+\t\t * hence, queue the current capture results as well. After that,\n+\t\t * check if any results are ready to be sent back to the\n+\t\t * framework.\n+\t\t */\n+\t\tdescriptor.status_ = Camera3RequestDescriptor::Succeeded;\n+\t\tdescriptor.resultMetadata_ = std::move(resultMetadata);\n+\t\tdescriptor.captureResult_ = captureResult;\n+\t\tstd::unique_ptr<Camera3RequestDescriptor> reqDescriptor =\n+\t\t\tstd::make_unique<Camera3RequestDescriptor>();\n+\t\t*reqDescriptor = std::move(descriptor);\n+\t\tqueuedDescriptor_.push_back(std::move(reqDescriptor));\n+\n+\t\tsendQueuedCaptureResults();\n+\t}\n+}\n+\n+void CameraDevice::streamProcessingComplete(CameraStream *cameraStream,\n+\t\t\t\t\t CameraStream::ProcessStatus status,\n+\t\t\t\t\t const Camera3RequestDescriptor *context)\n+{\n+\tfor (auto &d : queuedDescriptor_) {\n+\t\tif (d.get() != context)\n+\t\t\tcontinue;\n+\n \t\t/*\n \t\t * Return the FrameBuffer to the CameraStream now that we're\n \t\t * done processing it.\n \t\t */\n \t\tif (cameraStream->type() == CameraStream::Type::Internal)\n-\t\t\tcameraStream->putBuffer(src);\n-\n-\t\tif (ret) {\n-\t\t\tbuffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n-\t\t\tnotifyError(descriptor.frameNumber_, buffer.stream,\n-\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\t\t\tcameraStream->putBuffer(d->internalBuffer_);\n+\n+\t\tif (status == CameraStream::ProcessStatus::Succeeded) {\n+\t\t\td->status_ = Camera3RequestDescriptor::Succeeded;\n+\t\t} else {\n+\t\t\td->status_ = Camera3RequestDescriptor::Failed;\n+\n+\t\t\td->captureResult_.partial_result = 0;\n+\t\t\tfor (camera3_stream_buffer_t &buffer : d->buffers_) {\n+\t\t\t\tCameraStream *cs = static_cast<CameraStream *>(buffer.stream->priv);\n+\n+\t\t\t\tif (cs->camera3Stream().format != HAL_PIXEL_FORMAT_BLOB)\n+\t\t\t\t\tcontinue;\n+\t\t\t\tbuffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t\t\tnotifyError(d->frameNumber_, buffer.stream,\n+\t\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\t\t\t}\n \t\t}\n+\t\tbreak;\n \t}\n \n-\tcaptureResult.result = resultMetadata->get();\n-\tcallbacks_->process_capture_result(callbacks_, &captureResult);\n+\t/*\n+\t * Send back capture results to the framework by inspecting the queue.\n+\t * The framework can defer queueing further requests to the HAL (via\n+\t * process_capture_request) unless until it receives the capture results\n+\t * for already queued requests.\n+\t */\n+\tsendQueuedCaptureResults();\n+}\n+\n+void CameraDevice::sendQueuedCaptureResults()\n+{\n+\twhile (!queuedDescriptor_.empty()) {\n+\t\tstd::unique_ptr<Camera3RequestDescriptor> &d = queuedDescriptor_.front();\n+\t\tif (d->status_ == Camera3RequestDescriptor::Pending)\n+\t\t\treturn;\n+\n+\t\td->captureResult_.result = d->resultMetadata_->get();\n+\t\tcallbacks_->process_capture_result(callbacks_, &(d->captureResult_));\n+\t\tqueuedDescriptor_.pop_front();\n+\t}\n }\n \n std::string CameraDevice::logPrefix() const\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex b59ac3e7..e7318358 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -7,6 +7,7 @@\n #ifndef __ANDROID_CAMERA_DEVICE_H__\n #define __ANDROID_CAMERA_DEVICE_H__\n \n+#include <deque>\n #include <map>\n #include <memory>\n #include <mutex>\n@@ -46,6 +47,20 @@ struct Camera3RequestDescriptor {\n \tstd::vector<std::unique_ptr<libcamera::FrameBuffer>> frameBuffers_;\n \tCameraMetadata settings_;\n \tstd::unique_ptr<CaptureRequest> request_;\n+\n+\t/*\n+\t * Below are utility placeholders used when a capture result\n+\t * needs to be queued before completion via process_capture_result()\n+\t */\n+\tenum completionStatus {\n+\t\tPending,\n+\t\tSucceeded,\n+\t\tFailed,\n+\t};\n+\tstd::unique_ptr<CameraMetadata> resultMetadata_;\n+\tcamera3_capture_result_t captureResult_;\n+\tlibcamera::FrameBuffer *internalBuffer_;\n+\tcompletionStatus status_;\n };\n \n class CameraDevice : protected libcamera::Loggable\n@@ -78,6 +93,9 @@ public:\n \tint processCaptureRequest(camera3_capture_request_t *request);\n \tvoid requestComplete(libcamera::Request *request);\n \n+\tvoid streamProcessingComplete(CameraStream *stream,\n+\t\t\t\t CameraStream::ProcessStatus status,\n+\t\t\t\t const Camera3RequestDescriptor *context);\n protected:\n \tstd::string logPrefix() const override;\n \n@@ -106,6 +124,8 @@ private:\n \tstd::unique_ptr<CameraMetadata> getResultMetadata(\n \t\tconst Camera3RequestDescriptor &descriptor) const;\n \n+\tvoid sendQueuedCaptureResults();\n+\n \tunsigned int id_;\n \tcamera3_device_t camera3Device_;\n \n@@ -126,6 +146,8 @@ private:\n \tlibcamera::Mutex descriptorsMutex_; /* Protects descriptors_. */\n \tstd::map<uint64_t, Camera3RequestDescriptor> descriptors_;\n \n+\tstd::deque<std::unique_ptr<Camera3RequestDescriptor>> queuedDescriptor_;\n+\n \tstd::string maker_;\n \tstd::string model_;\n \ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex 996779c4..c7d874b2 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -134,6 +134,8 @@ void CameraStream::postProcessingComplete(PostProcessor::Status status,\n \t\tprocessStatus = ProcessStatus::Succeeded;\n \telse\n \t\tprocessStatus = ProcessStatus::Failed;\n+\n+\tcameraDevice_->streamProcessingComplete(this, processStatus, context);\n }\n \n FrameBuffer *CameraStream::getBuffer()\n", "prefixes": [ "libcamera-devel", "v2", "6/9" ] }