Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 14080,
    "url": "https://patchwork.libcamera.org/api/1.1/patches/14080/?format=api",
    "web_url": "https://patchwork.libcamera.org/patch/14080/",
    "project": {
        "id": 1,
        "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api",
        "name": "libcamera",
        "link_name": "libcamera",
        "list_id": "libcamera_core",
        "list_email": "libcamera-devel@lists.libcamera.org",
        "web_url": "",
        "scm_url": "",
        "webscm_url": ""
    },
    "msgid": "<20211011073505.243864-6-umang.jain@ideasonboard.com>",
    "date": "2021-10-11T07:35:03",
    "name": "[libcamera-devel,v4,5/7] android: post_processor: Make post processing async",
    "commit_ref": null,
    "pull_url": null,
    "state": "changes-requested",
    "archived": false,
    "hash": "96543365761910632f3d5ed611c2b9d68e9a715d",
    "submitter": {
        "id": 86,
        "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api",
        "name": "Umang Jain",
        "email": "umang.jain@ideasonboard.com"
    },
    "delegate": {
        "id": 12,
        "url": "https://patchwork.libcamera.org/api/1.1/users/12/?format=api",
        "username": "uajain",
        "first_name": "Umang",
        "last_name": "Jain",
        "email": "umang.jain@ideasonboard.com"
    },
    "mbox": "https://patchwork.libcamera.org/patch/14080/mbox/",
    "series": [
        {
            "id": 2611,
            "url": "https://patchwork.libcamera.org/api/1.1/series/2611/?format=api",
            "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2611",
            "date": "2021-10-11T07:34:58",
            "name": "Async Post Processor",
            "version": 4,
            "mbox": "https://patchwork.libcamera.org/series/2611/mbox/"
        }
    ],
    "comments": "https://patchwork.libcamera.org/api/patches/14080/comments/",
    "check": "pending",
    "checks": "https://patchwork.libcamera.org/api/patches/14080/checks/",
    "tags": {},
    "headers": {
        "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>",
        "X-Original-To": "parsemail@patchwork.libcamera.org",
        "Delivered-To": "parsemail@patchwork.libcamera.org",
        "Received": [
            "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id AC984C323E\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 11 Oct 2021 07:35:26 +0000 (UTC)",
            "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 72E2368F63;\n\tMon, 11 Oct 2021 09:35:26 +0200 (CEST)",
            "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9040568F5E\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 11 Oct 2021 09:35:24 +0200 (CEST)",
            "from perceval.ideasonboard.com (unknown [103.251.226.107])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 35EAE2BD;\n\tMon, 11 Oct 2021 09:35:23 +0200 (CEST)"
        ],
        "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"LOeODLNO\"; dkim-atps=neutral",
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1633937724;\n\tbh=lt8KIIhDPYVF4mqylcTznhAJjaGtCom4f7jvST+W90Q=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=LOeODLNOsR8M3MyjE1GwfwpL/r72oV33L+b4jOb+p5C7vk+10czu2F3sAJToWNm4Y\n\tK6SytLk8EaYHuyKff5ufdzd5JcEA1wuQbR/xXHKVqWfBf4VhlGMxCChQGUHayVYqNp\n\t8Qf7c4fWe4UuKgw7KPfb1xzrbd0RL5VLLYp56X4g=",
        "From": "Umang Jain <umang.jain@ideasonboard.com>",
        "To": "libcamera-devel@lists.libcamera.org",
        "Date": "Mon, 11 Oct 2021 13:05:03 +0530",
        "Message-Id": "<20211011073505.243864-6-umang.jain@ideasonboard.com>",
        "X-Mailer": "git-send-email 2.31.1",
        "In-Reply-To": "<20211011073505.243864-1-umang.jain@ideasonboard.com>",
        "References": "<20211011073505.243864-1-umang.jain@ideasonboard.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "Subject": "[libcamera-devel] [PATCH v4 5/7] android: post_processor: Make post\n\tprocessing async",
        "X-BeenThere": "libcamera-devel@lists.libcamera.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "<libcamera-devel.lists.libcamera.org>",
        "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>",
        "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>",
        "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>",
        "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>",
        "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>",
        "Errors-To": "libcamera-devel-bounces@lists.libcamera.org",
        "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"
    },
    "content": "Introduce a dedicated worker class derived from libcamera::Thread.\nThe worker class maintains a queue for post-processing requests\nand waits for a post-processing request to become available.\nIt will process them as per FIFO before de-queuing it from the\nqueue.\n\nTo get access to the source and destination buffers in the worker\nthread, we also need to save a pointer to them in the\nCamera3RequestDescriptor.\n\nThis patch also implements a flush() for the PostProcessorWorker\nclass which is responsible to purge post-processing requests\nqueued up while a camera is stopping/flushing.\n\nThe libcamera request completion handler CameraDevice::requestComplete()\nassumes that the request that has just completed is at the front of the\nqueue. Now that the post-processor runs asynchronously, this isn't true\nanymore, a request being post-processed will stay in the queue and a new\nlibcamera request may complete. Remove that assumption, and use the\nrequest cookie to obtain the Camera3RequestDescriptor.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\nSigned-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n---\n src/android/camera_device.cpp |  25 +-------\n src/android/camera_device.h   |   3 +\n src/android/camera_stream.cpp | 108 +++++++++++++++++++++++++++++++---\n src/android/camera_stream.h   |  39 +++++++++++-\n 4 files changed, 144 insertions(+), 31 deletions(-)",
    "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex eba370ea..61b902ad 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -239,6 +239,7 @@ Camera3RequestDescriptor::Camera3RequestDescriptor(\n \t/* Clone the controls associated with the camera3 request. */\n \tsettings_ = CameraMetadata(camera3Request->settings);\n \n+\tdest_.reset();\n \t/*\n \t * Create the CaptureRequest, stored as a unique_ptr<> to tie its\n \t * lifetime to the descriptor.\n@@ -1094,28 +1095,8 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n \n void CameraDevice::requestComplete(Request *request)\n {\n-\tCamera3RequestDescriptor *descriptor;\n-\t{\n-\t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tASSERT(!descriptors_.empty());\n-\t\tdescriptor = descriptors_.front().get();\n-\t}\n-\n-\tif (descriptor->request_->cookie() != request->cookie()) {\n-\t\t/*\n-\t\t * \\todo Clarify if the Camera has to be closed on\n-\t\t * ERROR_DEVICE and possibly demote the Fatal to simple\n-\t\t * Error.\n-\t\t */\n-\t\tnotifyError(0, nullptr, CAMERA3_MSG_ERROR_DEVICE);\n-\t\tLOG(HAL, Fatal)\n-\t\t\t<< \"Out-of-order completion for request \"\n-\t\t\t<< utils::hex(request->cookie());\n-\n-\t\tMutexLocker descriptorsLock(descriptorsMutex_);\n-\t\tdescriptors_.pop();\n-\t\treturn;\n-\t}\n+\tCamera3RequestDescriptor *descriptor =\n+\t\treinterpret_cast<Camera3RequestDescriptor *>(request->cookie());\n \n \t/*\n \t * Prepare the capture result for the Android camera stack.\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex eee97516..725a0618 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -59,6 +59,9 @@ struct Camera3RequestDescriptor {\n \tstd::unique_ptr<CameraMetadata> resultMetadata_;\n \tlibcamera::FrameBuffer *internalBuffer_;\n \n+\tstd::unique_ptr<CameraBuffer> dest_;\n+\tconst libcamera::FrameBuffer *src_;\n+\n \tcamera3_capture_result_t captureResult_ = {};\n \tStatus status_ = Status::Pending;\n };\ndiff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp\nindex cec07269..818ef948 100644\n--- a/src/android/camera_stream.cpp\n+++ b/src/android/camera_stream.cpp\n@@ -94,10 +94,12 @@ int CameraStream::configure()\n \t\tif (ret)\n \t\t\treturn ret;\n \n+\t\tworker_ = std::make_unique<PostProcessorWorker>(postProcessor_.get());\n \t\tpostProcessor_->processComplete.connect(\n \t\t\tthis, [&](Camera3RequestDescriptor *request, PostProcessor::Status status) {\n \t\t\t\tcameraDevice_->streamProcessingComplete(this, request, status);\n \t\t\t});\n+\t\tworker_->start();\n \t}\n \n \tif (type_ == Type::Internal) {\n@@ -167,19 +169,26 @@ void CameraStream::process(const FrameBuffer &source,\n \tif (!postProcessor_)\n \t\treturn;\n \n-\t/*\n-\t * \\todo Buffer mapping and processing should be moved to a\n-\t * separate thread.\n-\t */\n \tconst StreamConfiguration &output = configuration();\n-\tCameraBuffer dest(*camera3Dest.buffer, output.pixelFormat, output.size,\n-\t\t\t  PROT_READ | PROT_WRITE);\n-\tif (!dest.isValid()) {\n+\trequest->dest_ = std::make_unique<CameraBuffer>(\n+\t\t*camera3Dest.buffer, output.pixelFormat, output.size, PROT_READ | PROT_WRITE);\n+\tif (!request->dest_->isValid()) {\n \t\tLOG(HAL, Error) << \"Failed to create destination buffer\";\n \t\treturn;\n \t}\n \n-\tpostProcessor_->process(source, &dest, request);\n+\trequest->src_ = &source;\n+\n+\t/* Push the postProcessor request to the worker queue. */\n+\tworker_->queueRequest(request);\n+}\n+\n+void CameraStream::flush()\n+{\n+\tif (!postProcessor_)\n+\t\treturn;\n+\n+\tworker_->flush();\n }\n \n FrameBuffer *CameraStream::getBuffer()\n@@ -209,3 +218,86 @@ void CameraStream::putBuffer(FrameBuffer *buffer)\n \n \tbuffers_.push_back(buffer);\n }\n+\n+CameraStream::PostProcessorWorker::PostProcessorWorker(PostProcessor *postProcessor)\n+\t: postProcessor_(postProcessor)\n+{\n+}\n+\n+CameraStream::PostProcessorWorker::~PostProcessorWorker()\n+{\n+\t{\n+\t\tlibcamera::MutexLocker lock(mutex_);\n+\t\tstate_ = State::Stopped;\n+\t}\n+\n+\tcv_.notify_one();\n+\twait();\n+}\n+\n+void CameraStream::PostProcessorWorker::start()\n+{\n+\t{\n+\t\tlibcamera::MutexLocker lock(mutex_);\n+\t\tstate_ = State::Running;\n+\t}\n+\n+\tThread::start();\n+}\n+\n+void CameraStream::PostProcessorWorker::queueRequest(Camera3RequestDescriptor *request)\n+{\n+\t{\n+\t\tMutexLocker lock(mutex_);\n+\t\tASSERT(state_ == State::Running);\n+\t\trequests_.push(request);\n+\t}\n+\tcv_.notify_one();\n+}\n+\n+void CameraStream::PostProcessorWorker::run()\n+{\n+\tMutexLocker locker(mutex_);\n+\n+\twhile (1) {\n+\t\tcv_.wait(locker, [&] {\n+\t\t\treturn state_ != State::Running || !requests_.empty();\n+\t\t});\n+\n+\t\tif (state_ != State::Running)\n+\t\t\tbreak;\n+\n+\t\tCamera3RequestDescriptor *descriptor = requests_.front();\n+\t\trequests_.pop();\n+\t\tlocker.unlock();\n+\n+\t\tpostProcessor_->process(*descriptor->src_, descriptor->dest_.get(),\n+\t\t\t\t\tdescriptor);\n+\n+\t\tlocker.lock();\n+\t}\n+\n+\tif (state_ == State::Flushing) {\n+\t\twhile (!requests_.empty()) {\n+\t\t\tpostProcessor_->processComplete.emit(requests_.front(),\n+\t\t\t\t\t\t\t     PostProcessor::Status::Error);\n+\t\t\trequests_.pop();\n+\t\t}\n+\t\tstate_ = State::Stopped;\n+\t\tlocker.unlock();\n+\t\tcv_.notify_one();\n+\t}\n+}\n+\n+void CameraStream::PostProcessorWorker::flush()\n+{\n+\tlibcamera::MutexLocker lock(mutex_);\n+\tstate_ = State::Flushing;\n+\tlock.unlock();\n+\tcv_.notify_one();\n+\n+\tlock.lock();\n+\tcv_.wait(lock, [&] {\n+\t\treturn state_ == State::Stopped;\n+\t});\n+}\ndiff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\nindex a0c5f166..e410f35d 100644\n--- a/src/android/camera_stream.h\n+++ b/src/android/camera_stream.h\n@@ -7,21 +7,26 @@\n #ifndef __ANDROID_CAMERA_STREAM_H__\n #define __ANDROID_CAMERA_STREAM_H__\n \n+#include <condition_variable>\n #include <memory>\n #include <mutex>\n+#include <queue>\n #include <vector>\n \n #include <hardware/camera3.h>\n \n+#include <libcamera/base/thread.h>\n+\n #include <libcamera/camera.h>\n #include <libcamera/framebuffer.h>\n #include <libcamera/framebuffer_allocator.h>\n #include <libcamera/geometry.h>\n #include <libcamera/pixel_format.h>\n \n+#include \"post_processor.h\"\n+\n class CameraDevice;\n class CameraMetadata;\n-class PostProcessor;\n \n struct Camera3RequestDescriptor;\n \n@@ -125,8 +130,38 @@ public:\n \t\t     Camera3RequestDescriptor *request);\n \tlibcamera::FrameBuffer *getBuffer();\n \tvoid putBuffer(libcamera::FrameBuffer *buffer);\n+\tvoid flush();\n \n private:\n+\tclass PostProcessorWorker : public libcamera::Thread\n+\t{\n+\tpublic:\n+\t\tenum class State {\n+\t\t\tStopped,\n+\t\t\tRunning,\n+\t\t\tFlushing,\n+\t\t};\n+\n+\t\tPostProcessorWorker(PostProcessor *postProcessor);\n+\t\t~PostProcessorWorker();\n+\n+\t\tvoid start();\n+\t\tvoid queueRequest(Camera3RequestDescriptor *request);\n+\t\tvoid flush();\n+\n+\tprotected:\n+\t\tvoid run() override;\n+\n+\tprivate:\n+\t\tPostProcessor *postProcessor_;\n+\n+\t\tlibcamera::Mutex mutex_;\n+\t\tstd::condition_variable cv_;\n+\n+\t\tstd::queue<Camera3RequestDescriptor *> requests_;\n+\t\tState state_;\n+\t};\n+\n \tint waitFence(int fence);\n \n \tCameraDevice *const cameraDevice_;\n@@ -143,6 +178,8 @@ private:\n \t */\n \tstd::unique_ptr<std::mutex> mutex_;\n \tstd::unique_ptr<PostProcessor> postProcessor_;\n+\n+\tstd::unique_ptr<PostProcessorWorker> worker_;\n };\n \n #endif /* __ANDROID_CAMERA_STREAM__ */\n",
    "prefixes": [
        "libcamera-devel",
        "v4",
        "5/7"
    ]
}