From patchwork Tue Sep 7 19:57:04 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 13755 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 9F56EBDC71 for ; Tue, 7 Sep 2021 19:57:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 6F91B6916F; Tue, 7 Sep 2021 21:57:32 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="SwHjuvOC"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id B4C646916C for ; Tue, 7 Sep 2021 21:57:30 +0200 (CEST) Received: from perceval.ideasonboard.com (unknown [103.251.226.2]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id A198E499; Tue, 7 Sep 2021 21:57:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1631044650; bh=izciIO8uRaoT5jeJakvr4co67CIHwdBvEhRVlTOIgYo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=SwHjuvOCHqNEkxf52JoC7AdTrd5C2o1CLwjkOTJHd2bAO2ISNvQyke/IqU2P6H7us VZ56Umaeyy0dgJCXWiyx2VEG+J386geqB0Wbw7StqaVnUN20pPi+4RaCgzKumcL/yQ FIRHLQt9FtTXgSQqxOqMHnHyDomZrfcBL1pDLWds= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Wed, 8 Sep 2021 01:27:04 +0530 Message-Id: <20210907195704.338079-7-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20210907195704.338079-1-umang.jain@ideasonboard.com> References: <20210907195704.338079-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v1 6/6] android: camera_stream: Run post-processor in a separate thread X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In CameraStream, introduce a new private PostProcessorWorker class derived from Object. A instance of PostProcessorWorker is moved to a separate thread instance which will be responsible to run the post-processor. Running PostProcessor asynchronously should entail that all the data context needed by the PostProcessor should remain valid for the entire duration of its run. Most of the context preserving part has been addressed inthe previous patch, we just need to ensure the source framebuffer data that comes via Camera::Request, should remain valid for the entire duration of post-processing running asynchronously. In order to so, we maintain a separate copy of the framebuffer data and add it to the Camera3RequestDescriptor in which we preserve rest of the context. Signed-off-by: Umang Jain --- src/android/camera_device.cpp | 18 +++++++++++++++++- src/android/camera_device.h | 4 ++++ src/android/camera_stream.cpp | 15 ++++++++++++++- src/android/camera_stream.h | 26 ++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 8741678d..5a442fb9 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1173,6 +1173,21 @@ void CameraDevice::requestComplete(Request *request) return; } + FrameBuffer *source = src; + if (cameraStream->type() != CameraStream::Type::Internal) { + /* + * The source buffer is owned by Request object which + * can be reused for future capture request. Since + * post-processor will run asynchrnously, we need to + * copy the source buffer and use it as source buffer. + */ + for (const auto &plane : src->planes()) + descriptor.srcPlanes_.push_back(plane); + descriptor.srcFramebuffer_ = + std::make_unique(descriptor.srcPlanes_); + source = descriptor.srcFramebuffer_.get(); + } + std::unique_ptr reqDescriptor = std::make_unique(); *reqDescriptor = std::move(descriptor); @@ -1186,7 +1201,7 @@ void CameraDevice::requestComplete(Request *request) metadata); }); - int ret = cameraStream->process(src, currentDescriptor->destBuffer_.get(), + int ret = cameraStream->process(source, currentDescriptor->destBuffer_.get(), currentDescriptor->settings_, metadata); return; @@ -1260,6 +1275,7 @@ void CameraDevice::streamProcessingComplete(CameraStream *cameraStream, void CameraDevice::sendQueuedCaptureResults() { + MutexLocker lock(queuedDescriptorsMutex_); while (!queuedDescriptor_.empty()) { std::unique_ptr &d = queuedDescriptor_.front(); if (d->status_ != Camera3RequestDescriptor::NOT_FINISHED) { diff --git a/src/android/camera_device.h b/src/android/camera_device.h index 330c69a3..87ae00ab 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -100,6 +100,9 @@ private: camera3_capture_result_t captureResult_; libcamera::FrameBuffer *internalBuffer_; completionStatus status_; + + std::unique_ptr srcFramebuffer_; + std::vector srcPlanes_; }; enum class State { @@ -147,6 +150,7 @@ private: libcamera::Mutex descriptorsMutex_; /* Protects descriptors_. */ std::map descriptors_; + libcamera::Mutex queuedDescriptorsMutex_; /* Protects queuedDescriptor_. */ std::deque> queuedDescriptor_; std::string maker_; diff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp index 1939cb9c..6a2c1f5b 100644 --- a/src/android/camera_stream.cpp +++ b/src/android/camera_stream.cpp @@ -55,6 +55,15 @@ CameraStream::CameraStream(CameraDevice *const cameraDevice, * is what we instantiate here. */ postProcessor_ = std::make_unique(cameraDevice_); + ppWorker_ = std::make_unique(postProcessor_.get()); + + thread_ = std::make_unique(); + ppWorker_->moveToThread(thread_.get()); + /* + * \todo: Class is MoveConstructible, so where to stop thread + * if we don't user-defined destructor? + */ + thread_->start(); } if (type == Type::Internal) { @@ -108,7 +117,11 @@ int CameraStream::process(const FrameBuffer *source, if (!postProcessor_) return 0; - return postProcessor_->process(source, destBuffer, requestMetadata, resultMetadata); + ppWorker_->invokeMethod(&PostProcessorWorker::process, + ConnectionTypeQueued, source, destBuffer, + requestMetadata, resultMetadata); + + return 0; } void CameraStream::handlePostProcessing(PostProcessor::Status status) diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h index ad03e4cf..d8ce0972 100644 --- a/src/android/camera_stream.h +++ b/src/android/camera_stream.h @@ -13,7 +13,9 @@ #include +#include #include +#include #include #include @@ -23,6 +25,7 @@ #include "post_processor.h" +class CameraBuffer; class CameraDevice; class CameraMetadata; @@ -135,6 +138,27 @@ public: libcamera::Signal processComplete; private: + class PostProcessorWorker : public libcamera::Object + { + public: + PostProcessorWorker(PostProcessor *postProcessor) + { + postProcessor_ = postProcessor; + } + + void process(const libcamera::FrameBuffer *source, + CameraBuffer *destination, + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata) + { + postProcessor_->process(source, destination, + requestMetadata, resultMetadata); + } + + private: + PostProcessor *postProcessor_; + }; + void handlePostProcessing(PostProcessor::Status status); CameraDevice *const cameraDevice_; @@ -151,6 +175,8 @@ private: */ std::unique_ptr mutex_; std::unique_ptr postProcessor_; + std::unique_ptr ppWorker_; + std::unique_ptr thread_; }; #endif /* __ANDROID_CAMERA_STREAM__ */