Patch Detail
Show a patch.
GET /api/patches/16807/?format=api
{ "id": 16807, "url": "https://patchwork.libcamera.org/api/patches/16807/?format=api", "web_url": "https://patchwork.libcamera.org/patch/16807/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220726182500.425182-7-hanlinchen@chromium.org>", "date": "2022-07-26T18:25:00", "name": "[libcamera-devel,6/6] android: Implement partial result feature based on partialResultCompleted signal", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "2a1fb4c24fe50be3e9786fccbc7811b06fb4142b", "submitter": { "id": 98, "url": "https://patchwork.libcamera.org/api/people/98/?format=api", "name": "Hanlin Chen", "email": "hanlinchen@chromium.org" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/16807/mbox/", "series": [ { "id": 3329, "url": "https://patchwork.libcamera.org/api/series/3329/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=3329", "date": "2022-07-26T18:24:54", "name": "Implement Android Partial Result Featrue", "version": 1, "mbox": "https://patchwork.libcamera.org/series/3329/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/16807/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/16807/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4408BBE173\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 26 Jul 2022 18:25:42 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 00AFE63326;\n\tTue, 26 Jul 2022 20:25:42 +0200 (CEST)", "from mail-pl1-x62f.google.com (mail-pl1-x62f.google.com\n\t[IPv6:2607:f8b0:4864:20::62f])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 07DEB60487\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 26 Jul 2022 20:25:40 +0200 (CEST)", "by mail-pl1-x62f.google.com with SMTP id x7so169046pll.7\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 26 Jul 2022 11:25:39 -0700 (PDT)", "from localhost ([2401:fa00:1:17:739:6574:4032:5911])\n\tby smtp.gmail.com with UTF8SMTPSA id\n\te25-20020aa79819000000b0052aaff953aesm11923558pfl.115.2022.07.26.11.25.36\n\t(version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);\n\tTue, 26 Jul 2022 11:25:37 -0700 (PDT)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1658859942;\n\tbh=kC/obOEbRUaRNbcrPqopvP2rrKB/+JQCu9KyLBSHlK0=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=pEKf82/5OCeaka/LCWvx/SuvtJ0I/T6ldPUabtO1YWdtddRkgonpSaKMbdumQtz00\n\tMvP1qfl9aFqQeheJFE6yiTKts69yqGOXDIfg4MLAvnOgNto0+i6CTryy4mPg1sopXX\n\t1icfFS5hy5Obqq783IjsY/wSbFGk6wFEND/r/6v26xeBcLGHY4Dgy4pzMe2pm/ncXY\n\tQE5ZIwW/4d61edWndIgWDp8cGbSjHEYHY9JuqHGdiON3y6yRJjrTKHsw0yUz7MHuiO\n\tLDm1Pfm6N5fuMuAlac3B+rgFmvMcfo9hGQHZ8mXXur/7v7+3oIy96p61ZvymN1gPzA\n\t1VC7T3addQ+9Q==", "v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org;\n\ts=google; \n\th=from:to:cc:subject:date:message-id:in-reply-to:references\n\t:mime-version:content-transfer-encoding;\n\tbh=Q9AjxhyfkzdksonddqwU0JC18QtJBL2g9AqgOm7NYi4=;\n\tb=gAuWm/YvG2VrCEroIkL4QXy7PTqG4kFrTaE3WoV5y+076nSv3ZcvmhjVdlVBVqg01I\n\tMn+5EUoKCNyJTO4dlk9kda6gecx5FUZZBP0T35UXf1AxJEB3jA6Gi7zjQhtol0W/CQbf\n\t+O+P2LVnz+71tuIFK8uLgfM8si1hyJFdERgFA=" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=chromium.org\n\theader.i=@chromium.org header.b=\"gAuWm/Yv\"; \n\tdkim-atps=neutral", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:mime-version:content-transfer-encoding;\n\tbh=Q9AjxhyfkzdksonddqwU0JC18QtJBL2g9AqgOm7NYi4=;\n\tb=C9t+Ls03jeShg8x5CadzerDUDgBj9z7/RyvER05hCHhKwnfGyanKcrSas3MnGyyDG8\n\tLBqS3i6c0DEY/+1lqr+pyMUTRCemTai9b9EEzcPKWhl64UMoHWVk7RAmvHsz/9j65OMw\n\tburBeRFpBdAJ1DHjr7u+GVKONjh0RrbJegLJDzO88K9DXUBJGDniAgdaQxWqt/rJXqpE\n\tbXGJPr6jD8LYz+R5bqGzMivDeYeA7l2AJCokBcGyrPuBnvpo5rzRo591z7D6tcX3Ib7P\n\txjSzau1hayapu2DuLrqJti7fG8i2Kt/lzkr+9jzjeGGGPBIZYCX0aesYHWk50V08Crfd\n\tje1Q==", "X-Gm-Message-State": "AJIora/PnU9UFnIZyjRcSplltRZCpfqRfWSHAXafeZKqLVZJNk7ANOVI\n\tBrQVMIGTOYvVa5cMtm3C1tFqszJ6XokeYw==", "X-Google-Smtp-Source": "AGRyM1v6uXAmT/3AgvKSdiR+/UtTUev3Sjvn1spiqjDEFk/w0I/0cvlWR0sRvnJdyGqhBnmy/ffSQg==", "X-Received": "by 2002:a17:903:124f:b0:16b:8167:e34e with SMTP id\n\tu15-20020a170903124f00b0016b8167e34emr18140779plh.52.1658859937591; \n\tTue, 26 Jul 2022 11:25:37 -0700 (PDT)", "To": "libcamera-devel@lists.libcamera.org", "Date": "Wed, 27 Jul 2022 02:25:00 +0800", "Message-Id": "<20220726182500.425182-7-hanlinchen@chromium.org>", "X-Mailer": "git-send-email 2.37.1.359.gd136c6c3e2-goog", "In-Reply-To": "<20220726182500.425182-1-hanlinchen@chromium.org>", "References": "<20220726182500.425182-1-hanlinchen@chromium.org>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH 6/6] android: Implement partial result\n\tfeature based on partialResultCompleted signal", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Han-Lin Chen via libcamera-devel <libcamera-devel@lists.libcamera.org>", "Reply-To": "Han-Lin Chen <hanlinchen@chromium.org>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "To impelement partial result feature. The patch includes the following:\n1. Set Camera::RequestCompletionMode to Immediately.\n2. Return partial results on Camera::partialResultCompleted signal, and recycle\nrequests on Camera::requestCompleted.\n3. Refactory usage of pendingStreamsToProcess_ and starts post-processing on\npartial result receved.\n\nSigned-off-by: Han-Lin Chen <hanlinchen@chromium.org>\n---\n src/android/camera_capabilities.cpp | 2 +-\n src/android/camera_capabilities.h | 2 +\n src/android/camera_device.cpp | 687 ++++++++++++++---------\n src/android/camera_device.h | 23 +-\n src/android/camera_request.cpp | 50 +-\n src/android/camera_request.h | 46 +-\n src/android/jpeg/post_processor_jpeg.cpp | 2 +-\n 7 files changed, 522 insertions(+), 290 deletions(-)", "diff": "diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp\nindex 64bd8dde..6ed040d0 100644\n--- a/src/android/camera_capabilities.cpp\n+++ b/src/android/camera_capabilities.cpp\n@@ -1374,7 +1374,7 @@ int CameraCapabilities::initializeStaticMetadata()\n \tstaticMetadata_->addEntry(ANDROID_SCALER_CROPPING_TYPE, croppingType);\n \n \t/* Request static metadata. */\n-\tint32_t partialResultCount = 1;\n+\tint32_t partialResultCount = MaxMetadataPackIndex;\n \tstaticMetadata_->addEntry(ANDROID_REQUEST_PARTIAL_RESULT_COUNT,\n \t\t\t\t partialResultCount);\n \ndiff --git a/src/android/camera_capabilities.h b/src/android/camera_capabilities.h\nindex 6f66f221..b8fc20f1 100644\n--- a/src/android/camera_capabilities.h\n+++ b/src/android/camera_capabilities.h\n@@ -23,6 +23,8 @@\n class CameraCapabilities\n {\n public:\n+\tstatic constexpr int32_t MaxMetadataPackIndex = 64;\n+\n \tCameraCapabilities() = default;\n \n \tint initialize(std::shared_ptr<libcamera::Camera> camera,\ndiff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex a14d5de9..6d79fd34 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -24,9 +24,8 @@\n #include <libcamera/formats.h>\n #include <libcamera/property_ids.h>\n \n-#include \"system/graphics.h\"\n-\n #include \"camera_buffer.h\"\n+#include \"camera_capabilities.h\"\n #include \"camera_hal_config.h\"\n #include \"camera_ops.h\"\n #include \"camera_request.h\"\n@@ -249,7 +248,11 @@ CameraDevice::CameraDevice(unsigned int id, std::shared_ptr<Camera> camera)\n \t: id_(id), state_(State::Stopped), camera_(std::move(camera)),\n \t facing_(CAMERA_FACING_FRONT), orientation_(0)\n {\n+\t/* Set RequestCompletionMode to Immediately to send result early */\n+\tcamera_->setRequestCompletionMode(Camera::Immediately);\n+\n \tcamera_->requestCompleted.connect(this, &CameraDevice::requestComplete);\n+\tcamera_->partialResultCompleted.connect(this, &CameraDevice::partialResultComplete);\n \n \tmaker_ = \"libcamera\";\n \tmodel_ = \"cameraModel\";\n@@ -439,7 +442,7 @@ void CameraDevice::stop()\n \n \t{\n \t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tdescriptors_ = {};\n+\t\tdescriptors_.clear();\n \t}\n \n \tstreams_.clear();\n@@ -855,14 +858,30 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)\n \treturn 0;\n }\n \n-void CameraDevice::abortRequest(Camera3RequestDescriptor *descriptor) const\n+void CameraDevice::abortRequest(Camera3RequestDescriptor *descriptor)\n {\n-\tnotifyError(descriptor->frameNumber_, nullptr, CAMERA3_MSG_ERROR_REQUEST);\n+\tCamera3ResultDescriptor *result = new Camera3ResultDescriptor(descriptor);\n \n-\tfor (auto &buffer : descriptor->buffers_)\n-\t\tbuffer.status = StreamBuffer::Status::Error;\n+\tfor (auto &buffer : descriptor->buffers_) {\n+\t\tsetBufferStatus(buffer, StreamBuffer::Status::Error);\n+\t\tresult->buffers_.emplace_back(&buffer);\n+\t}\n \n-\tdescriptor->status_ = Camera3RequestDescriptor::Status::Error;\n+\t{\n+\t\tMutexLocker lock(descriptor->resultsMutex_);\n+\t\tdescriptor->status_ = Camera3RequestDescriptor::Status::Flushed;\n+\t\tdescriptor->results_.emplace_back(result);\n+\t}\n+\n+\t/*\n+\t * After CAMERA3_MSG_ERROR_REQUEST is notified, for a given frame,\n+\t * only process_capture_results with buffers in\n+\t * CAMERA3_BUFFER_STATUS_ERROR are allowed. No further notifys or\n+\t * process_capture_result with non-null metadata is allowed.\n+\t */\n+\tnotifyError(descriptor->frameNumber_, nullptr, CAMERA3_MSG_ERROR_REQUEST);\n+\n+\tcompleteResultDescriptor(result);\n }\n \n bool CameraDevice::isValidRequest(camera3_capture_request_t *camera3Request) const\n@@ -962,9 +981,9 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t * to a libcamera stream. Streams of type Mapped will be handled later.\n \t *\n \t * Collect the CameraStream associated to each requested capture stream.\n-\t * Since requestedStreams is an std:set<>, no duplications can happen.\n+\t * Since requestedStreams is an std:map<>, no duplications can happen.\n \t */\n-\tstd::set<CameraStream *> requestedStreams;\n+\tstd::map<CameraStream *, libcamera::FrameBuffer *> requestedStreams;\n \tfor (const auto &[i, buffer] : utils::enumerate(descriptor->buffers_)) {\n \t\tCameraStream *cameraStream = buffer.stream;\n \t\tcamera3_stream_t *camera3Stream = cameraStream->camera3Stream();\n@@ -983,8 +1002,6 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tFrameBuffer *frameBuffer = nullptr;\n \t\tUniqueFD acquireFence;\n \n-\t\tMutexLocker lock(descriptor->streamsProcessMutex_);\n-\n \t\tswitch (cameraStream->type()) {\n \t\tcase CameraStream::Type::Mapped:\n \t\t\t/* Mapped streams will be handled in the next loop. */\n@@ -1016,10 +1033,8 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\t */\n \t\t\tframeBuffer = cameraStream->getBuffer();\n \t\t\tbuffer.internalBuffer = frameBuffer;\n+\t\t\tbuffer.srcBuffer = frameBuffer;\n \t\t\tLOG(HAL, Debug) << ss.str() << \" (internal)\";\n-\n-\t\t\tdescriptor->pendingStreamsToProcess_.insert(\n-\t\t\t\t{ cameraStream, &buffer });\n \t\t\tbreak;\n \t\t}\n \n@@ -1032,7 +1047,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tdescriptor->request_->addBuffer(cameraStream->stream(),\n \t\t\t\t\t\tframeBuffer, std::move(fence));\n \n-\t\trequestedStreams.insert(cameraStream);\n+\t\trequestedStreams[cameraStream] = frameBuffer;\n \t}\n \n \t/*\n@@ -1054,29 +1069,38 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\t\t\t<< cameraStream->configuration().pixelFormat << \"]\"\n \t\t\t\t<< \" (mapped)\";\n \n-\t\tMutexLocker lock(descriptor->streamsProcessMutex_);\n-\t\tdescriptor->pendingStreamsToProcess_.insert({ cameraStream, &buffer });\n-\n \t\t/*\n \t\t * Make sure the CameraStream this stream is mapped on has been\n \t\t * added to the request.\n \t\t */\n \t\tCameraStream *sourceStream = cameraStream->sourceStream();\n+\n \t\tASSERT(sourceStream);\n-\t\tif (requestedStreams.find(sourceStream) != requestedStreams.end())\n+\t\tASSERT(sourceStream->type() == CameraStream::Type::Direct);\n+\n+\t\t/*\n+\t\t * If the buffer for the source stream is requested, use its\n+\t\t * framebuffer as the source buffer for post-processing.\n+\t\t */\n+\t\tauto iter = requestedStreams.find(sourceStream);\n+\t\tif (iter != requestedStreams.end()) {\n+\t\t\tbuffer.srcBuffer = iter->second;\n \t\t\tcontinue;\n+\t\t}\n \n \t\t/*\n-\t\t * If that's not the case, we need to add a buffer to the request\n-\t\t * for this stream.\n+\t\t * If that's not the case, we need to add an internal buffer\n+\t\t * to the request for this stream.\n+\t\t *\n+\t\t * \\todo Handle the case that multiple mapped streams need to\n+\t\t * create one internal buffer for the same source stream.\n \t\t */\n-\t\tFrameBuffer *frameBuffer = cameraStream->getBuffer();\n+\t\tFrameBuffer *frameBuffer = sourceStream->getBuffer();\n \t\tbuffer.internalBuffer = frameBuffer;\n+\t\tbuffer.srcBuffer = frameBuffer;\n \n \t\tdescriptor->request_->addBuffer(sourceStream->stream(),\n \t\t\t\t\t\tframeBuffer, nullptr);\n-\n-\t\trequestedStreams.erase(sourceStream);\n \t}\n \n \t/*\n@@ -1098,11 +1122,9 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \t\tCamera3RequestDescriptor *rawDescriptor = descriptor.get();\n \t\t{\n \t\t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\t\tdescriptors_.push(std::move(descriptor));\n+\t\t\tdescriptors_.push_back(std::move(descriptor));\n \t\t}\n \t\tabortRequest(rawDescriptor);\n-\t\tcompleteDescriptor(rawDescriptor);\n-\n \t\treturn 0;\n \t}\n \n@@ -1120,7 +1142,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \n \t{\n \t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tdescriptors_.push(std::move(descriptor));\n+\t\tdescriptors_.push_back(std::move(descriptor));\n \t}\n \n \tcamera_->queueRequest(request);\n@@ -1128,131 +1150,238 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \treturn 0;\n }\n \n-void CameraDevice::requestComplete(Request *request)\n+void CameraDevice::partialResultComplete(Request *request, Result *result)\n {\n \tCamera3RequestDescriptor *descriptor =\n \t\treinterpret_cast<Camera3RequestDescriptor *>(request->cookie());\n \n-\t/*\n-\t * Prepare the capture result for the Android camera stack.\n-\t *\n-\t * The buffer status is set to Success and later changed to Error if\n-\t * post-processing/compression fails.\n-\t */\n-\tfor (auto &buffer : descriptor->buffers_) {\n-\t\tCameraStream *stream = buffer.stream;\n+\tif (result->buffers().empty() && result->metadata().empty())\n+\t\tLOG(HAL, Fatal)\n+\t\t\t<< \"Partial result should have at least buffer or metadata \"\n+\t\t\t<< descriptor->frameNumber_;\n \n+\tCamera3ResultDescriptor *camera3Result = new Camera3ResultDescriptor(descriptor);\n+\n+\tconst ControlList &metadata = result->metadata();\n+\tif (!metadata.empty()) {\n \t\t/*\n-\t\t * Streams of type Direct have been queued to the\n-\t\t * libcamera::Camera and their acquire fences have\n-\t\t * already been waited on by the library.\n-\t\t *\n-\t\t * Acquire fences of streams of type Internal and Mapped\n-\t\t * will be handled during post-processing.\n+\t\t * Notify shutter as soon as we have received SensorTimestamp.\n \t\t */\n-\t\tif (stream->type() == CameraStream::Type::Direct) {\n-\t\t\t/* If handling of the fence has failed restore buffer.fence. */\n-\t\t\tstd::unique_ptr<Fence> fence = buffer.frameBuffer->releaseFence();\n-\t\t\tif (fence)\n-\t\t\t\tbuffer.fence = fence->release();\n+\n+\t\tconst auto ×tamp = metadata.get(controls::SensorTimestamp);\n+\t\tif (timestamp) {\n+\t\t\tnotifyShutter(descriptor->frameNumber_, *timestamp);\n+\t\t\tLOG(HAL, Debug) << \"Request \" << request->cookie() << \" notifies shutter\";\n \t\t}\n-\t\tbuffer.status = StreamBuffer::Status::Success;\n+\n+\t\tcamera3Result->resultMetadata_ = getDynamicResultMetadata(metadata);\n \t}\n \n-\t/*\n-\t * If the Request has failed, abort the request by notifying the error\n-\t * and complete the request with all buffers in error state.\n-\t */\n-\tif (request->status() != Request::RequestComplete) {\n-\t\tLOG(HAL, Error) << \"Request \" << request->cookie()\n-\t\t\t\t<< \" not successfully completed: \"\n-\t\t\t\t<< request->status();\n+\tMutexLocker locker(camera3Result->streamsProcessMutex_);\n \n-\t\tabortRequest(descriptor);\n-\t\tcompleteDescriptor(descriptor);\n+\tfor (auto &buffer : descriptor->buffers_) {\n+\t\tCameraStream *cameraStream = buffer.stream;\n+\t\tfor (auto *frameBuffer : result->buffers()) {\n+\t\t\tif (buffer.srcBuffer != frameBuffer &&\n+\t\t\t buffer.frameBuffer.get() != frameBuffer)\n+\t\t\t\tcontinue;\n \n-\t\treturn;\n+\t\t\tbuffer.result = camera3Result;\n+\t\t\tcamera3Result->buffers_.emplace_back(&buffer);\n+\n+\t\t\tStreamBuffer::Status status = StreamBuffer::Status::Success;\n+\t\t\tif (frameBuffer->metadata().status != FrameMetadata::FrameSuccess) {\n+\t\t\t\tstatus = StreamBuffer::Status::Error;\n+\t\t\t}\n+\t\t\tsetBufferStatus(buffer, status);\n+\n+\t\t\tswitch (cameraStream->type()) {\n+\t\t\tcase CameraStream::Type::Direct: {\n+\t\t\t\tASSERT(buffer.frameBuffer.get() == frameBuffer);\n+\t\t\t\t/*\n+\t\t\t\t * Streams of type Direct have been queued to the\n+\t\t\t\t * libcamera::Camera and their acquire fences have\n+\t\t\t\t * already been waited on by the library.\n+\t\t\t\t */\n+\t\t\t\tstd::unique_ptr<Fence> fence = buffer.frameBuffer->releaseFence();\n+\t\t\t\tif (fence)\n+\t\t\t\t\tbuffer.fence = fence->release();\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tcase CameraStream::Type::Mapped:\n+\t\t\tcase CameraStream::Type::Internal:\n+\t\t\t\tASSERT(buffer.srcBuffer == frameBuffer);\n+\t\t\t\tif (status == StreamBuffer::Status::Error) {\n+\t\t\t\t\t/*\n+\t\t\t\t\t * If the framebuffer is internal to CameraStream return\n+\t\t\t\t\t * it back now that we're done processing it.\n+\t\t\t\t\t */\n+\t\t\t\t\tif (buffer.internalBuffer)\n+\t\t\t\t\t\treturnInternalBuffer(buffer);\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\n+\t\t\t\t/*\n+\t\t\t\t * Acquire fences of streams of type Internal and Mapped\n+\t\t\t\t * will be handled during post-processing.\n+\t\t\t\t */\n+\t\t\t\tbuffer.srcBuffer = frameBuffer;\n+\t\t\t\tcamera3Result->pendingBuffersToProcess_.emplace_back(&buffer);\n+\n+\t\t\t\tif (cameraStream->isJpegStream()) {\n+\t\t\t\t\tgenerateJpegExifMetadata(descriptor, &buffer);\n+\n+\t\t\t\t\t/*\n+\t\t\t\t\t * Allocate for post-processor to fill\n+\t\t\t\t\t * in JPEG related metadata.\n+\t\t\t\t\t */\n+\t\t\t\t\tif (!camera3Result->resultMetadata_)\n+\t\t\t\t\t\tcamera3Result->resultMetadata_ = getDynamicResultMetadata(metadata);\n+\t\t\t\t}\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t{\n+\t\t/*\n+\t\t * Adding result to the request before sending it to the\n+\t\t * post-processing threads, so the streamProcessingComplete()\n+\t\t * slot can safely call compeleteResult().\n+\t\t */\n+\t\tMutexLocker lock(descriptor->resultsMutex_);\n+\t\tdescriptor->results_.emplace_back(camera3Result);\n \t}\n \n \t/*\n-\t * Notify shutter as soon as we have verified we have a valid request.\n-\t *\n-\t * \\todo The shutter event notification should be sent to the framework\n-\t * as soon as possible, earlier than request completion time.\n+\t * Queue all the post-processing streams request at once. The completion\n+\t * slot streamProcessingComplete() can only execute when we are out\n+\t * this critical section (result->streamsProcessMutex_). This helps to\n+\t * handle synchronous errors here itself.\n \t */\n-\tuint64_t sensorTimestamp = static_cast<uint64_t>(request->metadata()\n-\t\t\t\t\t\t\t\t .get(controls::SensorTimestamp)\n-\t\t\t\t\t\t\t\t .value_or(0));\n-\tnotifyShutter(descriptor->frameNumber_, sensorTimestamp);\n+\tfor (auto *buffer : camera3Result->pendingBuffersToProcess_) {\n+\t\tint ret = buffer->stream->process(buffer);\n+\t\tif (ret) {\n+\t\t\tsetBufferStatus(*buffer, StreamBuffer::Status::Error);\n+\n+\t\t\t/*\n+\t\t\t * If the framebuffer is internal to CameraStream return\n+\t\t\t * it back now that we're done processing it.\n+\t\t\t */\n+\t\t\tif (buffer->internalBuffer)\n+\t\t\t\tbuffer->stream->putBuffer(buffer->internalBuffer);\n \n-\tLOG(HAL, Debug) << \"Request \" << request->cookie() << \" completed with \"\n-\t\t\t<< descriptor->request_->buffers().size() << \" streams\";\n+\t\t\tLOG(HAL, Error) << \"Failed to run post process of request \" << descriptor->frameNumber_;\n+\t\t}\n+\t}\n+\n+\tif (!camera3Result->pendingBuffersToProcess_.empty())\n+\t\treturn;\n+\n+\tlocker.unlock();\n+\tcompleteResultDescriptor(camera3Result);\n+}\n+\n+void CameraDevice::requestComplete(Request *request)\n+{\n+\tCamera3RequestDescriptor *camera3Request =\n+\t\treinterpret_cast<Camera3RequestDescriptor *>(request->cookie());\n \n \t/*\n-\t * Generate the metadata associated with the captured buffers.\n+\t * On Android, each new partial result with a metadata must set a\n+\t * field (partial_result) to a distinct inclusive value between\n+\t * 1 and ANDROID_REQUEST_PARTIAL_RESULT_COUNT and the final result with\n+\t * metadata has to set the field as ANDROID_REQUEST_PARTIAL_RESULT_COUNT.\n+\t *\n+\t * An empty metadata with ANDROID_REQUEST_PARTIAL_RESULT_COUNT is not\n+\t * allowed. Add a result with a fixed metadata on requestComplete()\n+\t * in case all of previous metadata are sent early, and no more metadata\n+\t * can be sent as the final with ANDROID_REQUEST_PARTIAL_RESULT_COUNT.\n \t *\n-\t * Notify if the metadata generation has failed, but continue processing\n-\t * buffers and return an empty metadata pack.\n+\t * See comments in completeResultDescriptor().\n \t */\n-\tdescriptor->resultMetadata_ = getResultMetadata(*descriptor);\n-\tif (!descriptor->resultMetadata_) {\n-\t\tnotifyError(descriptor->frameNumber_, nullptr, CAMERA3_MSG_ERROR_RESULT);\n+\tCamera3ResultDescriptor *result = new Camera3ResultDescriptor(camera3Request);\n+\tresult->resultMetadata_ = getFixedResultMetadata(camera3Request->settings_);\n \n-\t\t/*\n-\t\t * The camera framework expects an empty metadata pack on error.\n-\t\t *\n-\t\t * \\todo Check that the post-processor code handles this situation\n-\t\t * correctly.\n-\t\t */\n-\t\tdescriptor->resultMetadata_ = std::make_unique<CameraMetadata>(0, 0);\n+\t{\n+\t\tMutexLocker lock(camera3Request->resultsMutex_);\n+\n+\t\tswitch (request->status()) {\n+\t\tcase Request::RequestComplete:\n+\t\t\tcamera3Request->status_ = Camera3RequestDescriptor::Status::Success;\n+\t\t\tbreak;\n+\t\tcase Request::RequestCancelled:\n+\t\t\tcamera3Request->status_ = Camera3RequestDescriptor::Status::Cancelled;\n+\t\t\tbreak;\n+\t\tcase Request::RequestPending:\n+\t\t\tLOG(HAL, Fatal) << \"Try to complete an unfinished request\";\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tcamera3Request->results_.emplace_back(result);\n \t}\n \n-\t/* Handle post-processing. */\n-\tMutexLocker locker(descriptor->streamsProcessMutex_);\n+\tcompleteResultDescriptor(result);\n+\treturn;\n+}\n+\n+void CameraDevice::completeResultDescriptor(Camera3ResultDescriptor *result)\n+{\n+\tCamera3RequestDescriptor *request = result->request_;\n+\tresult->complete_ = true;\n+\n+\tMutexLocker lock(request->resultsMutex_);\n+\n+\tbool requestCompleted = (request->status_ != Camera3RequestDescriptor::Status::Pending);\n+\tbool requestCancelled = (request->status_ == Camera3RequestDescriptor::Status::Cancelled);\n+\n+\tbool hasPendingMetadata = false;\n+\tbool hasPendingResult = false;\n+\n+\tfor (auto &r : request->results_) {\n+\t\tif (!r->complete_) {\n+\t\t\thasPendingResult = true;\n+\t\t\tif (r->resultMetadata_)\n+\t\t\t\thasPendingMetadata = true;\n+\t\t}\n+\t}\n \n \t/*\n-\t * Queue all the post-processing streams request at once. The completion\n-\t * slot streamProcessingComplete() can only execute when we are out\n-\t * this critical section. This helps to handle synchronous errors here\n-\t * itself.\n+\t * Android requires value of metadataPackIndex follows the rules:\n+\t *\n+\t * Set to 0: Result contains no metadata (buffers only).\n+\t * Set between 1 and (MaxMetadataPackIndex - 1): Result contains metadata.\n+\t * Set to MaxMetadataPackIndex: The final result having metadata.\n \t */\n-\tauto iter = descriptor->pendingStreamsToProcess_.begin();\n-\twhile (iter != descriptor->pendingStreamsToProcess_.end()) {\n-\t\tCameraStream *stream = iter->first;\n-\t\tStreamBuffer *buffer = iter->second;\n+\tuint32_t &metadataPackIndex = result->metadataPackIndex_ = 0;\n+\tif (result->resultMetadata_) {\n+\t\tmetadataPackIndex = request->nextPartialResultIndex_++;\n \n-\t\tif (stream->isJpegStream()) {\n-\t\t\tgenerateJpegExifMetadata(descriptor, buffer);\n+\t\tif (requestCompleted && !hasPendingMetadata) {\n+\t\t\tmetadataPackIndex = CameraCapabilities::MaxMetadataPackIndex;\n \t\t}\n-\n-\t\tFrameBuffer *src = request->findBuffer(stream->stream());\n-\t\tif (!src) {\n-\t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n-\t\t\tsetBufferStatus(*buffer, StreamBuffer::Status::Error);\n-\t\t\titer = descriptor->pendingStreamsToProcess_.erase(iter);\n-\t\t\tcontinue;\n+\t\tif (metadataPackIndex > CameraCapabilities::MaxMetadataPackIndex) {\n+\t\t\tLOG(HAL, Fatal) << \"Partial result exceed limited count \"\n+\t\t\t\t\t<< CameraCapabilities::MaxMetadataPackIndex;\n \t\t}\n+\t}\n \n-\t\tbuffer->srcBuffer = src;\n+\tlock.unlock();\n \n-\t\t++iter;\n-\t\tint ret = stream->process(buffer);\n-\t\tif (ret) {\n-\t\t\tsetBufferStatus(*buffer, StreamBuffer::Status::Error);\n-\t\t\tdescriptor->pendingStreamsToProcess_.erase(stream);\n+\tsendCaptureResult(result);\n \n-\t\t\t/*\n-\t\t\t * If the framebuffer is internal to CameraStream return\n-\t\t\t * it back now that we're done processing it.\n-\t\t\t */\n-\t\t\tif (buffer->internalBuffer)\n-\t\t\t\tstream->putBuffer(buffer->internalBuffer);\n-\t\t}\n+\t/*\n+\t * Call notify with CAMERA3_MSG_ERROR_RESULT to indicate some of the\n+\t * expected result metadata might not be available for the capture. Only\n+\t * calls when all pending metadata are sent, since Android ignores\n+\t * the following metadata after the notice.\n+\t */\n+\tif (requestCancelled && !hasPendingMetadata) {\n+\t\tnotifyError(request->frameNumber_, nullptr, CAMERA3_MSG_ERROR_RESULT);\n \t}\n \n-\tif (descriptor->pendingStreamsToProcess_.empty()) {\n-\t\tlocker.unlock();\n-\t\tcompleteDescriptor(descriptor);\n+\tif (requestCompleted && !hasPendingResult) {\n+\t\tcompleteRequestDescriptor(request);\n \t}\n }\n \n@@ -1260,91 +1389,99 @@ void CameraDevice::requestComplete(Request *request)\n * \\brief Complete the Camera3RequestDescriptor\n * \\param[in] descriptor The Camera3RequestDescriptor that has completed\n *\n- * The function marks the Camera3RequestDescriptor as 'complete'. It shall be\n- * called when all the streams in the Camera3RequestDescriptor have completed\n- * capture (or have been generated via post-processing) and the request is ready\n- * to be sent back to the framework.\n+ * The function shall be called when all the buffers and metadata in the\n+ * Camera3RequestDescriptor have completed (or have been generated via\n+ * post-processing) and all results are sent back to the framework.\n *\n * \\context This function is \\threadsafe.\n */\n-void CameraDevice::completeDescriptor(Camera3RequestDescriptor *descriptor)\n+void CameraDevice::completeRequestDescriptor(Camera3RequestDescriptor *descriptor)\n {\n \tMutexLocker lock(descriptorsMutex_);\n-\tdescriptor->complete_ = true;\n \n-\tsendCaptureResults();\n+\tauto iter = find_if(descriptors_.begin(), descriptors_.end(),\n+\t\t\t [&descriptor](auto &item) {\n+\t\t\t\t return item.get() == descriptor;\n+\t\t\t });\n+\n+\tif (iter == descriptors_.end()) {\n+\t\tLOG(HAL, Error) << \"Complete an unrecognized request\";\n+\t\treturn;\n+\t}\n+\n+\tdescriptors_.erase(iter);\n }\n \n-/**\n- * \\brief Sequentially send capture results to the framework\n- *\n- * Iterate over the descriptors queue to send completed descriptors back to the\n- * framework, in the same order as they have been queued. For each complete\n- * descriptor, populate a locally-scoped camera3_capture_result_t from the\n- * descriptor, send the capture result back by calling the\n- * process_capture_result() callback, and remove the descriptor from the queue.\n- * Stop iterating if the descriptor at the front of the queue is not complete.\n- *\n- * This function should never be called directly in the codebase. Use\n- * completeDescriptor() instead.\n- */\n-void CameraDevice::sendCaptureResults()\n+void CameraDevice::setBufferStatus(StreamBuffer &streamBuffer,\n+\t\t\t\t StreamBuffer::Status status) const\n {\n-\twhile (!descriptors_.empty() && !descriptors_.front()->isPending()) {\n-\t\tauto descriptor = std::move(descriptors_.front());\n-\t\tdescriptors_.pop();\n+\tstreamBuffer.status = status;\n+\tif (status != StreamBuffer::Status::Success) {\n+\t\tnotifyError(streamBuffer.request->frameNumber_,\n+\t\t\t streamBuffer.stream->camera3Stream(),\n+\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\t}\n+}\n \n-\t\tcamera3_capture_result_t captureResult = {};\n+void CameraDevice::returnInternalBuffer(StreamBuffer &streamBuffer) const\n+{\n+\tif (!streamBuffer.internalBuffer) {\n+\t\treturn;\n+\t}\n \n-\t\tcaptureResult.frame_number = descriptor->frameNumber_;\n+\tCameraStream *stream = streamBuffer.stream;\n+\tswitch (stream->type()) {\n+\tcase CameraStream::Type::Internal:\n+\t\tstream->putBuffer(streamBuffer.internalBuffer);\n+\t\tbreak;\n+\tcase CameraStream::Type::Mapped:\n+\t\t/*\n+\t\t * The internal buffer of a mapped stream is allocated from\n+\t\t * its source stream\n+\t\t */\n+\t\tstream->sourceStream()->putBuffer(streamBuffer.internalBuffer);\n+\t\tbreak;\n+\tcase CameraStream::Type::Direct:\n+\t\tLOG(HAL, Fatal) << \"Direct stream buffer has internal buffer\";\n+\t\tbreak;\n+\t}\n+}\n \n-\t\tif (descriptor->resultMetadata_)\n-\t\t\tcaptureResult.result =\n-\t\t\t\tdescriptor->resultMetadata_->getMetadata();\n+void CameraDevice::sendCaptureResult(Camera3ResultDescriptor *result) const\n+{\n+\tcamera3_capture_result_t captureResult = {};\n \n-\t\tstd::vector<camera3_stream_buffer_t> resultBuffers;\n-\t\tresultBuffers.reserve(descriptor->buffers_.size());\n+\tcaptureResult.frame_number = result->request_->frameNumber_;\n \n-\t\tfor (auto &buffer : descriptor->buffers_) {\n-\t\t\tcamera3_buffer_status status = CAMERA3_BUFFER_STATUS_ERROR;\n+\tstd::vector<camera3_stream_buffer_t> resultBuffers;\n+\tresultBuffers.reserve(result->buffers_.size());\n \n-\t\t\tif (buffer.status == StreamBuffer::Status::Success)\n-\t\t\t\tstatus = CAMERA3_BUFFER_STATUS_OK;\n+\tfor (auto &buffer : result->buffers_) {\n+\t\tcamera3_buffer_status status = CAMERA3_BUFFER_STATUS_ERROR;\n \n-\t\t\t/*\n-\t\t\t * Pass the buffer fence back to the camera framework as\n-\t\t\t * a release fence. This instructs the framework to wait\n-\t\t\t * on the acquire fence in case we haven't done so\n-\t\t\t * ourselves for any reason.\n-\t\t\t */\n-\t\t\tresultBuffers.push_back({ buffer.stream->camera3Stream(),\n-\t\t\t\t\t\t buffer.camera3Buffer, status,\n-\t\t\t\t\t\t -1, buffer.fence.release() });\n-\t\t}\n+\t\tif (buffer->status == StreamBuffer::Status::Success)\n+\t\t\tstatus = CAMERA3_BUFFER_STATUS_OK;\n \n-\t\tcaptureResult.num_output_buffers = resultBuffers.size();\n-\t\tcaptureResult.output_buffers = resultBuffers.data();\n+\t\t/*\n+\t\t * Pass the buffer fence back to the camera framework as\n+\t\t * a release fence. This instructs the framework to wait\n+\t\t * on the acquire fence in case we haven't done so\n+\t\t * ourselves for any reason.\n+\t\t */\n+\t\tresultBuffers.push_back({ buffer->stream->camera3Stream(),\n+\t\t\t\t\t buffer->camera3Buffer, status,\n+\t\t\t\t\t -1, buffer->fence.release() });\n+\t}\n \n-\t\tif (descriptor->status_ == Camera3RequestDescriptor::Status::Success)\n-\t\t\tcaptureResult.partial_result = 1;\n+\tcaptureResult.num_output_buffers = resultBuffers.size();\n+\tcaptureResult.output_buffers = resultBuffers.data();\n \n-\t\tcallbacks_->process_capture_result(callbacks_, &captureResult);\n-\t}\n-}\n+\tif (result->resultMetadata_)\n+\t\tcaptureResult.result = result->resultMetadata_->getMetadata();\n \n-void CameraDevice::setBufferStatus(StreamBuffer &streamBuffer,\n-\t\t\t\t StreamBuffer::Status status)\n-{\n-\tstreamBuffer.status = status;\n-\tif (status != StreamBuffer::Status::Success) {\n-\t\tnotifyError(streamBuffer.request->frameNumber_,\n-\t\t\t streamBuffer.stream->camera3Stream(),\n-\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\tcaptureResult.partial_result = result->metadataPackIndex_;\n \n-\t\t/* Also set error status on entire request descriptor. */\n-\t\tstreamBuffer.request->status_ =\n-\t\t\tCamera3RequestDescriptor::Status::Error;\n-\t}\n+\tcallbacks_->process_capture_result(callbacks_, &captureResult);\n }\n \n /**\n@@ -1354,11 +1491,11 @@ void CameraDevice::setBufferStatus(StreamBuffer &streamBuffer,\n *\n * This function is called from the post-processor's thread whenever a camera\n * stream has finished post processing. The corresponding entry is dropped from\n- * the descriptor's pendingStreamsToProcess_ map.\n+ * the result's pendingBufferToProcess_ list.\n *\n- * If the pendingStreamsToProcess_ map is then empty, all streams requiring to\n- * be generated from post-processing have been completed. Mark the descriptor as\n- * complete using completeDescriptor() in that case.\n+ * If the pendingBufferToProcess_ list is then empty, all streams requiring to\n+ * be generated from post-processing have been completed. Mark the result as\n+ * complete using completeResultDescriptor() in that case.\n */\n void CameraDevice::streamProcessingComplete(StreamBuffer *streamBuffer,\n \t\t\t\t\t StreamBuffer::Status status)\n@@ -1370,19 +1507,18 @@ void CameraDevice::streamProcessingComplete(StreamBuffer *streamBuffer,\n \t * that we're done processing it.\n \t */\n \tif (streamBuffer->internalBuffer)\n-\t\tstreamBuffer->stream->putBuffer(streamBuffer->internalBuffer);\n-\n-\tCamera3RequestDescriptor *request = streamBuffer->request;\n+\t\treturnInternalBuffer(*streamBuffer);\n \n+\tCamera3ResultDescriptor *result = streamBuffer->result;\n \t{\n-\t\tMutexLocker locker(request->streamsProcessMutex_);\n+\t\tMutexLocker locker(result->streamsProcessMutex_);\n+\t\tresult->pendingBuffersToProcess_.remove(streamBuffer);\n \n-\t\trequest->pendingStreamsToProcess_.erase(streamBuffer->stream);\n-\t\tif (!request->pendingStreamsToProcess_.empty())\n+\t\tif (!result->pendingBuffersToProcess_.empty())\n \t\t\treturn;\n \t}\n \n-\tcompleteDescriptor(streamBuffer->request);\n+\tcompleteResultDescriptor(streamBuffer->result);\n }\n \n std::string CameraDevice::logPrefix() const\n@@ -1414,6 +1550,99 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n \tcallbacks_->notify(callbacks_, ¬ify);\n }\n \n+std::unique_ptr<CameraMetadata>\n+CameraDevice::getDynamicResultMetadata(const ControlList &metadata) const\n+{\n+\t/*\n+\t * \\todo Keep this in sync with the actual number of entries.\n+\t *\n+\t * Reserve capacity for the metadata larger than 4 bytes which cannot\n+\t * store in entries.\n+\t * Currently: 6 entries, 40 bytes extra capaticy.\n+\t *\n+\t * ANDROID_SENSOR_TIMESTAMP (int64) = 8 bytes\n+\t * ANDROID_SENSOR_EXPOSURE_TIME (int64) = 8 bytes\n+\t * ANDROID_SENSOR_FRAME_DURATION (int64) = 8 bytes\n+\t * ANDROID_SCALER_CROP_REGION (int32 X 4) = 16 bytes\n+\t * Total bytes for capacity: 40\n+\t *\n+\t * Reserve more capacity for the JPEG metadata set by the post-processor.\n+\t * Currently: 8 entries, 72 bytes extra capaticy.\n+\t *\n+\t * ANDROID_JPEG_GPS_COORDINATES (double x 3) = 24 bytes\n+\t * ANDROID_JPEG_GPS_PROCESSING_METHOD (byte x 32) = 32 bytes\n+\t * ANDROID_JPEG_GPS_TIMESTAMP (int64) = 8 bytes\n+\t * ANDROID_JPEG_SIZE (int32_t) = 4 bytes\n+\t * ANDROID_JPEG_QUALITY (byte) = 1 byte\n+\t * ANDROID_JPEG_ORIENTATION (int32_t) = 4 bytes\n+\t * ANDROID_JPEG_THUMBNAIL_QUALITY (byte) = 1 byte\n+\t * ANDROID_JPEG_THUMBNAIL_SIZE (int32 x 2) = 8 bytes\n+\t * Total bytes for JPEG metadata: 72\n+\t *\n+\t * \\todo Calculate the entries and capacity by the input ControlList.\n+\t */\n+\tstd::unique_ptr<CameraMetadata> resultMetadata =\n+\t\tstd::make_unique<CameraMetadata>(14, 112);\n+\tif (!resultMetadata->isValid()) {\n+\t\tLOG(HAL, Error) << \"Failed to allocate result metadata\";\n+\t\treturn nullptr;\n+\t}\n+\n+\t/* Add metadata tags reported by libcamera. */\n+\tconst auto ×tamp = metadata.get(controls::SensorTimestamp);\n+\tif (timestamp)\n+\t\tresultMetadata->addEntry(ANDROID_SENSOR_TIMESTAMP, *timestamp);\n+\n+\tconst auto &pipelineDepth = metadata.get(controls::draft::PipelineDepth);\n+\tif (pipelineDepth)\n+\t\tresultMetadata->addEntry(ANDROID_REQUEST_PIPELINE_DEPTH,\n+\t\t\t\t\t *pipelineDepth);\n+\n+\tconst auto &exposureTime = metadata.get(controls::ExposureTime);\n+\tif (exposureTime)\n+\t\tresultMetadata->addEntry(ANDROID_SENSOR_EXPOSURE_TIME,\n+\t\t\t\t\t *exposureTime * 1000ULL);\n+\n+\tconst auto &frameDuration = metadata.get(controls::FrameDuration);\n+\tif (frameDuration)\n+\t\tresultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,\n+\t\t\t\t\t *frameDuration * 1000);\n+\n+\tconst auto &scalerCrop = metadata.get(controls::ScalerCrop);\n+\tif (scalerCrop) {\n+\t\tconst Rectangle &crop = *scalerCrop;\n+\t\tint32_t cropRect[] = {\n+\t\t\tcrop.x,\n+\t\t\tcrop.y,\n+\t\t\tstatic_cast<int32_t>(crop.width),\n+\t\t\tstatic_cast<int32_t>(crop.height),\n+\t\t};\n+\t\tresultMetadata->addEntry(ANDROID_SCALER_CROP_REGION, cropRect);\n+\t}\n+\n+\tconst auto &testPatternMode = metadata.get(controls::draft::TestPatternMode);\n+\tif (testPatternMode)\n+\t\tresultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE,\n+\t\t\t\t\t *testPatternMode);\n+\n+\t/*\n+\t * Return the result metadata pack even is not valid: get() will return\n+\t * nullptr.\n+\t */\n+\tif (!resultMetadata->isValid()) {\n+\t\tLOG(HAL, Error) << \"Failed to construct result metadata\";\n+\t}\n+\n+\tif (resultMetadata->resized()) {\n+\t\tauto [entryCount, dataCount] = resultMetadata->usage();\n+\t\tLOG(HAL, Info)\n+\t\t\t<< \"Result metadata resized: \" << entryCount\n+\t\t\t<< \" entries and \" << dataCount << \" bytes used\";\n+\t}\n+\n+\treturn resultMetadata;\n+}\n+\n /*\n * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n */\n@@ -1424,10 +1653,8 @@ void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n \tauto &jpegExifMetadata = buffer->jpegExifMetadata;\n \tjpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n \n-\tjpegExifMetadata->sensorExposureTime = 0;\n-\tif (metadata.contains(controls::ExposureTime)) {\n-\t\tjpegExifMetadata->sensorExposureTime = metadata.get(controls::ExposureTime) * 1000ULL;\n-\t}\n+\tconst int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n+\tjpegExifMetadata->sensorExposureTime = exposureTime;\n \n \t/*\n \t * todo: Android Sensitivity = analog gain X digital gain only on sensor.\n@@ -1441,31 +1668,26 @@ void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n * Produce a set of fixed result metadata.\n */\n std::unique_ptr<CameraMetadata>\n-CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) const\n+CameraDevice::getFixedResultMetadata(const CameraMetadata &settings) const\n {\n-\tconst ControlList &metadata = descriptor.request_->metadata();\n-\tconst CameraMetadata &settings = descriptor.settings_;\n \tcamera_metadata_ro_entry_t entry;\n \tbool found;\n \n \t/*\n+\t * \\todo Retrieve metadata from corresponding libcamera controls.\n \t * \\todo Keep this in sync with the actual number of entries.\n-\t * Currently: 40 entries, 156 bytes\n \t *\n-\t * Reserve more space for the JPEG metadata set by the post-processor.\n-\t * Currently:\n-\t * ANDROID_JPEG_GPS_COORDINATES (double x 3) = 24 bytes\n-\t * ANDROID_JPEG_GPS_PROCESSING_METHOD (byte x 32) = 32 bytes\n-\t * ANDROID_JPEG_GPS_TIMESTAMP (int64) = 8 bytes\n-\t * ANDROID_JPEG_SIZE (int32_t) = 4 bytes\n-\t * ANDROID_JPEG_QUALITY (byte) = 1 byte\n-\t * ANDROID_JPEG_ORIENTATION (int32_t) = 4 bytes\n-\t * ANDROID_JPEG_THUMBNAIL_QUALITY (byte) = 1 byte\n-\t * ANDROID_JPEG_THUMBNAIL_SIZE (int32 x 2) = 8 bytes\n-\t * Total bytes for JPEG metadata: 82\n+\t * Reserve capacity for the metadata larger than 4 bytes which cannot\n+\t * store in entries.\n+\t * Currently: 31 entries, 16 bytes\n+\t *\n+\t * ANDROID_CONTROL_AE_TARGET_FPS_RANGE (int32 X 2) = 8 bytes\n+\t * ANDROID_SENSOR_ROLLING_SHUTTER_SKEW (int64) = 8 bytes\n+\t *\n+\t * Total bytes: 16\n \t */\n \tstd::unique_ptr<CameraMetadata> resultMetadata =\n-\t\tstd::make_unique<CameraMetadata>(88, 166);\n+\t\tstd::make_unique<CameraMetadata>(31, 16);\n \tif (!resultMetadata->isValid()) {\n \t\tLOG(HAL, Error) << \"Failed to allocate result metadata\";\n \t\treturn nullptr;\n@@ -1563,9 +1785,6 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons\n \tresultMetadata->addEntry(ANDROID_LENS_OPTICAL_STABILIZATION_MODE,\n \t\t\t\t value);\n \n-\tvalue32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;\n-\tresultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);\n-\n \tvalue = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;\n \tresultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);\n \n@@ -1587,40 +1806,6 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons\n \tresultMetadata->addEntry(ANDROID_SENSOR_ROLLING_SHUTTER_SKEW,\n \t\t\t\t rolling_shutter_skew);\n \n-\t/* Add metadata tags reported by libcamera. */\n-\tconst int64_t timestamp = metadata.get(controls::SensorTimestamp).value_or(0);\n-\tresultMetadata->addEntry(ANDROID_SENSOR_TIMESTAMP, timestamp);\n-\n-\tconst auto &pipelineDepth = metadata.get(controls::draft::PipelineDepth);\n-\tif (pipelineDepth)\n-\t\tresultMetadata->addEntry(ANDROID_REQUEST_PIPELINE_DEPTH,\n-\t\t\t\t\t *pipelineDepth);\n-\n-\tconst auto &exposureTime = metadata.get(controls::ExposureTime);\n-\tif (exposureTime)\n-\t\tresultMetadata->addEntry(ANDROID_SENSOR_EXPOSURE_TIME,\n-\t\t\t\t\t *exposureTime * 1000ULL);\n-\n-\tconst auto &frameDuration = metadata.get(controls::FrameDuration);\n-\tif (frameDuration)\n-\t\tresultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,\n-\t\t\t\t\t *frameDuration * 1000);\n-\n-\tconst auto &scalerCrop = metadata.get(controls::ScalerCrop);\n-\tif (scalerCrop) {\n-\t\tconst Rectangle &crop = *scalerCrop;\n-\t\tint32_t cropRect[] = {\n-\t\t\tcrop.x, crop.y, static_cast<int32_t>(crop.width),\n-\t\t\tstatic_cast<int32_t>(crop.height),\n-\t\t};\n-\t\tresultMetadata->addEntry(ANDROID_SCALER_CROP_REGION, cropRect);\n-\t}\n-\n-\tconst auto &testPatternMode = metadata.get(controls::draft::TestPatternMode);\n-\tif (testPatternMode)\n-\t\tresultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE,\n-\t\t\t\t\t *testPatternMode);\n-\n \t/*\n \t * Return the result metadata pack even is not valid: get() will return\n \t * nullptr.\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 7b279895..0c4237a6 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -20,6 +20,7 @@\n #include <libcamera/base/mutex.h>\n \n #include <libcamera/camera.h>\n+#include <libcamera/controls.h>\n #include <libcamera/framebuffer.h>\n #include <libcamera/geometry.h>\n #include <libcamera/pixel_format.h>\n@@ -29,7 +30,6 @@\n #include \"camera_capabilities.h\"\n #include \"camera_metadata.h\"\n #include \"camera_stream.h\"\n-#include \"jpeg/encoder.h\"\n \n class Camera3RequestDescriptor;\n struct CameraConfigData;\n@@ -63,6 +63,8 @@ public:\n \tconst camera_metadata_t *constructDefaultRequestSettings(int type);\n \tint configureStreams(camera3_stream_configuration_t *stream_list);\n \tint processCaptureRequest(camera3_capture_request_t *request);\n+\tvoid partialResultComplete(libcamera::Request *request,\n+\t\t\t\t libcamera::Result *result);\n \tvoid requestComplete(libcamera::Request *request);\n \tvoid streamProcessingComplete(StreamBuffer *bufferStream,\n \t\t\t\t StreamBuffer::Status status);\n@@ -87,20 +89,25 @@ private:\n \tcreateFrameBuffer(const buffer_handle_t camera3buffer,\n \t\t\t libcamera::PixelFormat pixelFormat,\n \t\t\t const libcamera::Size &size);\n-\tvoid abortRequest(Camera3RequestDescriptor *descriptor) const;\n \tbool isValidRequest(camera3_capture_request_t *request) const;\n \tvoid notifyShutter(uint32_t frameNumber, uint64_t timestamp);\n \tvoid notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n \t\t\t camera3_error_msg_code code) const;\n \tint processControls(Camera3RequestDescriptor *descriptor);\n-\tvoid completeDescriptor(Camera3RequestDescriptor *descriptor)\n+\tvoid abortRequest(Camera3RequestDescriptor *descriptor);\n+\tvoid completeResultDescriptor(Camera3ResultDescriptor *result);\n+\tvoid completeRequestDescriptor(Camera3RequestDescriptor *descriptor)\n \t\tLIBCAMERA_TSA_EXCLUDES(descriptorsMutex_);\n-\tvoid sendCaptureResults() LIBCAMERA_TSA_REQUIRES(descriptorsMutex_);\n-\tvoid setBufferStatus(StreamBuffer &buffer, StreamBuffer::Status status);\n+\tvoid sendCaptureResult(Camera3ResultDescriptor *partialResult) const;\n+\tvoid setBufferStatus(StreamBuffer &buffer, StreamBuffer::Status status) const;\n+\tvoid returnInternalBuffer(StreamBuffer &buffer) const;\n \tvoid generateJpegExifMetadata(Camera3RequestDescriptor *request,\n \t\t\t\t StreamBuffer *buffer) const;\n-\tstd::unique_ptr<CameraMetadata> getResultMetadata(\n-\t\tconst Camera3RequestDescriptor &descriptor) const;\n+\n+\tstd::unique_ptr<CameraMetadata> getDynamicResultMetadata(\n+\t\tconst libcamera::ControlList &metadata) const;\n+\tstd::unique_ptr<CameraMetadata> getFixedResultMetadata(\n+\t\tconst CameraMetadata &settings) const;\n \n \tunsigned int id_;\n \tcamera3_device_t camera3Device_;\n@@ -118,7 +125,7 @@ private:\n \tstd::vector<CameraStream> streams_;\n \n \tlibcamera::Mutex descriptorsMutex_ LIBCAMERA_TSA_ACQUIRED_AFTER(stateMutex_);\n-\tstd::queue<std::unique_ptr<Camera3RequestDescriptor>> descriptors_\n+\tstd::list<std::unique_ptr<Camera3RequestDescriptor>> descriptors_\n \t\tLIBCAMERA_TSA_GUARDED_BY(descriptorsMutex_);\n \n \tstd::string maker_;\ndiff --git a/src/android/camera_request.cpp b/src/android/camera_request.cpp\nindex f5d4d314..31751b5c 100644\n--- a/src/android/camera_request.cpp\n+++ b/src/android/camera_request.cpp\n@@ -42,25 +42,25 @@ using namespace libcamera;\n * │ processCaptureRequest(camera3_capture_request_t request) │\n * │ │\n * │ - Create Camera3RequestDescriptor tracking this request │\n- * │ - Streams requiring post-processing are stored in the │\n- * │ pendingStreamsToProcess map │\n+ * │ - Buffer requiring post-processing are marked by the │\n+ * │ CameraStream::Type as Mapped or Internal │\n * │ - Add this Camera3RequestDescriptor to descriptors' queue │\n * │ CameraDevice::descriptors_ │\n- * │ │ ┌─────────────────────────┐\n- * │ - Queue the capture request to libcamera core ────────────┤►│libcamera core │\n- * │ │ ├─────────────────────────┤\n- * │ │ │- Capture from Camera │\n- * │ │ │ │\n- * │ │ │- Emit │\n- * │ │ │ Camera::requestComplete│\n- * │ requestCompleted(Request *request) ◄───────────────────────┼─┼──── │\n- * │ │ │ │\n- * │ - Check request completion status │ └─────────────────────────┘\n+ * │ │ ┌───────────────────────────────┐\n+ * │ - Queue the capture request to libcamera core ────────────┤►│libcamera core │\n+ * │ │ ├───────────────────────────────┤\n+ * │ │ │- Capture from Camera │\n+ * │ │ │ │\n+ * │ │ │- Emit │\n+ * │ │ │ Camera::partialResultComplete│\n+ * │ partialResultComplete(Request *request, Result result*) ◄──┼─┼──── │\n+ * │ │ │ │\n+ * │ - Check request completion status │ └───────────────────────────────┘\n * │ │\n- * │ - if (pendingStreamsToProcess > 0) │\n- * │ Queue all entries from pendingStreamsToProcess │\n+ * │ - if (pendingBuffersToProcess > 0) │\n+ * │ Queue all entries from pendingBuffersToProcess │\n * │ else │ │\n- * │ completeDescriptor() │ └──────────────────────┐\n+ * │ completeResultDescriptor() │ └──────────────────────┐\n * │ │ │\n * │ ┌──────────────────────────┴───┬──────────────────┐ │\n * │ │ │ │ │\n@@ -93,10 +93,10 @@ using namespace libcamera;\n * │ | | | | │\n * │ | - Check and set buffer status | | .... | │\n * │ | - Remove post+processing entry | | | │\n- * │ | from pendingStreamsToProcess | | | │\n+ * │ | from pendingBuffersToProcess | | | │\n * │ | | | | │\n- * │ | - if (pendingStreamsToProcess.empty())| | | │\n- * │ | completeDescriptor | | | │\n+ * │ | - if (pendingBuffersToProcess.empty())| | | │\n+ * │ | completeResultDescriptor | | | │\n * │ | | | | │\n * │ +---------------------------------------+ +--------------+ │\n * │ │\n@@ -110,6 +110,7 @@ using namespace libcamera;\n \n Camera3RequestDescriptor::Camera3RequestDescriptor(\n \tCamera *camera, const camera3_capture_request_t *camera3Request)\n+\t: status_(Status::Pending), nextPartialResultIndex_(1)\n {\n \tframeNumber_ = camera3Request->frame_number;\n \n@@ -140,6 +141,19 @@ Camera3RequestDescriptor::Camera3RequestDescriptor(\n \n Camera3RequestDescriptor::~Camera3RequestDescriptor() = default;\n \n+/*\n+ * \\class Camera3ResultDescriptor\n+ *\n+ * A utility class that groups information about a capture result to be later\n+ * sent to framework.\n+ */\n+Camera3ResultDescriptor::Camera3ResultDescriptor(Camera3RequestDescriptor *request)\n+\t: complete_(false), request_(request), metadataPackIndex_(1)\n+{\n+}\n+\n+Camera3ResultDescriptor::~Camera3ResultDescriptor() = default;\n+\n /**\n * \\class StreamBuffer\n * \\brief Group information for per-stream buffer of Camera3RequestDescriptor\ndiff --git a/src/android/camera_request.h b/src/android/camera_request.h\nindex f91de955..b5d9e3ee 100644\n--- a/src/android/camera_request.h\n+++ b/src/android/camera_request.h\n@@ -24,6 +24,7 @@\n \n class CameraBuffer;\n class CameraStream;\n+class Camera3ResultDescriptor;\n class Camera3RequestDescriptor;\n \n class StreamBuffer\n@@ -56,41 +57,64 @@ public:\n \tconst libcamera::FrameBuffer *srcBuffer = nullptr;\n \tstd::unique_ptr<CameraBuffer> dstBuffer;\n \tstd::optional<JpegExifMetadata> jpegExifMetadata;\n+\tCamera3ResultDescriptor *result;\n \tCamera3RequestDescriptor *request;\n \n private:\n \tLIBCAMERA_DISABLE_COPY(StreamBuffer)\n };\n \n+class Camera3ResultDescriptor\n+{\n+public:\n+\tCamera3ResultDescriptor(Camera3RequestDescriptor *request);\n+\t~Camera3ResultDescriptor();\n+\n+\tbool complete_;\n+\tCamera3RequestDescriptor *request_;\n+\tuint32_t metadataPackIndex_;\n+\n+\tstd::unique_ptr<CameraMetadata> resultMetadata_;\n+\tstd::vector<StreamBuffer *> buffers_;\n+\n+\t/* Keeps track of buffers waiting for post-processing. */\n+\tstd::list<StreamBuffer *> pendingBuffersToProcess_\n+\t\tLIBCAMERA_TSA_GUARDED_BY(streamsProcessMutex_);\n+\tlibcamera::Mutex streamsProcessMutex_;\n+\n+private:\n+\tLIBCAMERA_DISABLE_COPY(Camera3ResultDescriptor)\n+};\n+\n class Camera3RequestDescriptor\n {\n public:\n \tenum class Status {\n+\t\tPending,\n \t\tSuccess,\n-\t\tError,\n+\t\tCancelled,\n+\t\tFlushed,\n \t};\n \n-\t/* Keeps track of streams requiring post-processing. */\n-\tstd::map<CameraStream *, StreamBuffer *> pendingStreamsToProcess_\n-\t\tLIBCAMERA_TSA_GUARDED_BY(streamsProcessMutex_);\n-\tlibcamera::Mutex streamsProcessMutex_;\n-\n \tCamera3RequestDescriptor(libcamera::Camera *camera,\n \t\t\t\t const camera3_capture_request_t *camera3Request);\n \t~Camera3RequestDescriptor();\n \n-\tbool isPending() const { return !complete_; }\n-\n \tuint32_t frameNumber_ = 0;\n \n \tstd::vector<StreamBuffer> buffers_;\n \n \tCameraMetadata settings_;\n \tstd::unique_ptr<libcamera::Request> request_;\n-\tstd::unique_ptr<CameraMetadata> resultMetadata_;\n \n-\tbool complete_ = false;\n-\tStatus status_ = Status::Success;\n+\tstd::vector<std::unique_ptr<Camera3ResultDescriptor>> results_\n+\t\tLIBCAMERA_TSA_GUARDED_BY(resultsMutex_);\n+\n+\tStatus status_ LIBCAMERA_TSA_GUARDED_BY(resultsMutex_);\n+\n+\tuint32_t nextPartialResultIndex_ LIBCAMERA_TSA_GUARDED_BY(resultsMutex_);\n+\n+\tlibcamera::Mutex resultsMutex_;\n \n private:\n \tLIBCAMERA_DISABLE_COPY(Camera3RequestDescriptor)\ndiff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\nindex 91fd329c..0cb5fd9a 100644\n--- a/src/android/jpeg/post_processor_jpeg.cpp\n+++ b/src/android/jpeg/post_processor_jpeg.cpp\n@@ -111,7 +111,7 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n \tASSERT(jpegExifMetadata.has_value());\n \n \tconst CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n-\tCameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n+\tCameraMetadata *resultMetadata = streamBuffer->result->resultMetadata_.get();\n \tcamera_metadata_ro_entry_t entry;\n \tint ret;\n \n", "prefixes": [ "libcamera-devel", "6/6" ] }