From patchwork Fri Sep 10 07:06:37 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 13806 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 1E01FBDC71 for ; Fri, 10 Sep 2021 07:07:05 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id B84126917B; Fri, 10 Sep 2021 09:07:04 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Js9BbFM9"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 9700B6916E for ; Fri, 10 Sep 2021 09:07:02 +0200 (CEST) Received: from perceval.ideasonboard.com (unknown [103.251.226.149]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 8DB4B883; Fri, 10 Sep 2021 09:07:01 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1631257622; bh=GhFCseGpcp3egLXZPZSTCi008bvqr/gET9XO/EDdE6c=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Js9BbFM9nHF3wCdXq8Ssm5eGBmvtxPNNLA0CfNNHnsPIfsE9R93fvc0OYRPUwBC/U +v6omaGw14mwKTiNw0nKs/Y6OXhmFQdEc+7ni/T7buN5w4c0dVOqvDKM2WUBEsreQJ hEXw+7FND291MHWPnf2XR22p8H5QNsf2shsQu4QI= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Fri, 10 Sep 2021 12:36:37 +0530 Message-Id: <20210910070638.467294-9-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20210910070638.467294-1-umang.jain@ideasonboard.com> References: <20210910070638.467294-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 8/9] android: camera_stream: Run post-processor in a separate thread X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In CameraStream, introduce a new private PostProcessorWorker class derived from Object. A instance of PostProcessorWorker is moved to a separate thread instance which will be responsible to run the post-processor. Running PostProcessor asynchronously should entail that all the data context needed by the PostProcessor should remain valid for the entire duration of its run. Most of the context preserving part has been addressed in the previous commits, we just need to ensure the source framebuffer data that comes via Camera::Request, should remain valid for the entire duration of post-processing running asynchronously. In order to so, we maintain a separate copy of the framebuffer data and add it to the Camera3RequestDescriptor structure in which we preserve rest of the context. Signed-off-by: Umang Jain --- src/android/camera_device.cpp | 19 ++++++++++++++++++- src/android/camera_device.h | 4 ++++ src/android/camera_stream.cpp | 16 ++++++++++++++-- src/android/camera_stream.h | 28 ++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 988d4232..73eb5758 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1174,13 +1174,29 @@ void CameraDevice::requestComplete(Request *request) return; } + FrameBuffer *source = src; + if (cameraStream->type() != CameraStream::Type::Internal) { + /* + * The source buffer is owned by Request object which + * can be reused by libcamera. Since post-processor will + * run asynchrnously, we need to copy the request's + * frame buffer and use that as the source buffer for + * post processing. + */ + for (const auto &plane : src->planes()) + descriptor.srcPlanes_.push_back(plane); + descriptor.srcFramebuffer_ = + std::make_unique(descriptor.srcPlanes_); + source = descriptor.srcFramebuffer_.get(); + } + std::unique_ptr reqDescriptor = std::make_unique(); *reqDescriptor = std::move(descriptor); queuedDescriptor_.push_back(std::move(reqDescriptor)); Camera3RequestDescriptor *currentDescriptor = queuedDescriptor_.back().get(); - int ret = cameraStream->process(src, + int ret = cameraStream->process(source, currentDescriptor->destBuffer_.get(), currentDescriptor->settings_, currentDescriptor->resultMetadata_.get(), @@ -1255,6 +1271,7 @@ void CameraDevice::streamProcessingComplete(CameraStream *cameraStream, void CameraDevice::sendQueuedCaptureResults() { + MutexLocker lock(queuedDescriptorsMutex_); while (!queuedDescriptor_.empty()) { std::unique_ptr &d = queuedDescriptor_.front(); if (d->status_ == Camera3RequestDescriptor::Pending) diff --git a/src/android/camera_device.h b/src/android/camera_device.h index b62d373c..ecdda06e 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -62,6 +62,9 @@ struct Camera3RequestDescriptor { camera3_capture_result_t captureResult_; libcamera::FrameBuffer *internalBuffer_; completionStatus status_; + + std::unique_ptr srcFramebuffer_; + std::vector srcPlanes_; }; class CameraDevice : protected libcamera::Loggable @@ -147,6 +150,7 @@ private: libcamera::Mutex descriptorsMutex_; /* Protects descriptors_. */ std::map descriptors_; + libcamera::Mutex queuedDescriptorsMutex_; /* Protects queuedDescriptor_. */ std::deque> queuedDescriptor_; std::string maker_; diff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp index 5fd04bbf..845e2462 100644 --- a/src/android/camera_stream.cpp +++ b/src/android/camera_stream.cpp @@ -55,6 +55,15 @@ CameraStream::CameraStream(CameraDevice *const cameraDevice, * is what we instantiate here. */ postProcessor_ = std::make_unique(cameraDevice_); + ppWorker_ = std::make_unique(postProcessor_.get()); + + thread_ = std::make_unique(); + ppWorker_->moveToThread(thread_.get()); + /* + * \todo: Class is MoveConstructible, so where to stop thread + * if we don't user-defined destructor? See RFC patch at the end. + */ + thread_->start(); } if (type == Type::Internal) { @@ -110,8 +119,11 @@ int CameraStream::process(const FrameBuffer *source, if (!postProcessor_) return 0; - return postProcessor_->process(source, destBuffer, requestMetadata, - resultMetadata, context); + ppWorker_->invokeMethod(&PostProcessorWorker::process, + ConnectionTypeQueued, source, destBuffer, + requestMetadata, resultMetadata, context); + + return 0; } void CameraStream::postProcessingComplete(PostProcessor::Status status, diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h index 8097ddbc..dbb7eee3 100644 --- a/src/android/camera_stream.h +++ b/src/android/camera_stream.h @@ -13,7 +13,9 @@ #include +#include #include +#include #include #include @@ -23,6 +25,7 @@ #include "post_processor.h" +class CameraBuffer; class CameraDevice; class CameraMetadata; @@ -137,6 +140,29 @@ public: }; private: + class PostProcessorWorker : public libcamera::Object + { + public: + PostProcessorWorker(PostProcessor *postProcessor) + { + postProcessor_ = postProcessor; + } + + void process(const libcamera::FrameBuffer *source, + CameraBuffer *destination, + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata, + const Camera3RequestDescriptor *context) + { + postProcessor_->process(source, destination, + requestMetadata, resultMetadata, + context); + } + + private: + PostProcessor *postProcessor_; + }; + void postProcessingComplete(PostProcessor::Status status, const Camera3RequestDescriptor *context); @@ -154,6 +180,8 @@ private: */ std::unique_ptr mutex_; std::unique_ptr postProcessor_; + std::unique_ptr ppWorker_; + std::unique_ptr thread_; }; #endif /* __ANDROID_CAMERA_STREAM__ */