From patchwork Wed Oct 20 10:42:09 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 14189 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id AE842BF415 for ; Wed, 20 Oct 2021 10:42:23 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3C3A168F5C; Wed, 20 Oct 2021 12:42:23 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="sKOZbJlI"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 450D568F58 for ; Wed, 20 Oct 2021 12:42:21 +0200 (CEST) Received: from perceval.ideasonboard.com (unknown [103.251.226.185]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 40DE62A5; Wed, 20 Oct 2021 12:42:20 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1634726541; bh=1n5eo1ItDKoPwgz5CdzMWKGu7I6QstwOrPYxpoX+msc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=sKOZbJlIMLwmUqFzjrLxI6fa11HD4iGK7RJpkVK95eVBJJGhnmWCfRCxWU7AFTcbL FP4D9S565JnugL6JV7i+3N/2A2xWzq63aFyXAzCwB0b79RIjaaRnTP/lHZ+dTG0Ych 8/akaMJDIro4O4dP9udRQi2UD5kdlfaa4fiARmmI= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Wed, 20 Oct 2021 16:12:09 +0530 Message-Id: <20211020104212.121743-2-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20211020104212.121743-1-umang.jain@ideasonboard.com> References: <20211020104212.121743-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 1/4] android: Notify post processing completion via a signal X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Notify that the post processing for a request has been completed, via a signal. A pointer to the descriptor which is tracking the capture request is emitted along with the status of post processed buffer. The function CameraDevice::streamProcessingComplete() will finally set the status on the request descriptor and send capture results back to the framework accordingly. We also need to save a pointer to any internal buffers that might have been allocated by CameraStream. The buffer should be returned back to CameraStream just before capture results are sent. A streamProcessMutex_ has been introduced here itself, which will be applicable to guard access to descriptor->buffers_ when post-processing is moved to be asynchronous in subsequent commits. Signed-off-by: Umang Jain --- src/android/camera_device.cpp | 92 +++++++++++++++++++++--- src/android/camera_device.h | 7 ++ src/android/camera_request.h | 4 ++ src/android/camera_stream.cpp | 13 ++++ src/android/jpeg/post_processor_jpeg.cpp | 2 + src/android/post_processor.h | 9 +++ src/android/yuv/post_processor_yuv.cpp | 10 ++- 7 files changed, 125 insertions(+), 12 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 806b4090..541c2c81 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1117,6 +1117,15 @@ void CameraDevice::requestComplete(Request *request) } /* Handle post-processing. */ + bool needsPostProcessing = false; + Camera3RequestDescriptor::Status processingStatus = + Camera3RequestDescriptor::Status::Pending; + /* + * \todo Apply streamsProcessMutex_ when post-processing is adapted to run + * asynchronously. If we introduce the streamsProcessMutex_ right away, the + * lock will be held forever since it is synchronous at this point + * (see streamProcessingComplete). + */ for (auto &buffer : descriptor->buffers_) { CameraStream *stream = buffer.stream; @@ -1132,22 +1141,27 @@ void CameraDevice::requestComplete(Request *request) continue; } - int ret = stream->process(*src, buffer, descriptor); - - /* - * Return the FrameBuffer to the CameraStream now that we're - * done processing it. - */ if (stream->type() == CameraStream::Type::Internal) - stream->putBuffer(src); + buffer.internalBuffer = src; + needsPostProcessing = true; + int ret = stream->process(*src, buffer, descriptor); if (ret) { - buffer.status = Camera3RequestDescriptor::Status::Error; - notifyError(descriptor->frameNumber_, stream->camera3Stream(), - CAMERA3_MSG_ERROR_BUFFER); + setBufferStatus(stream, buffer, descriptor, + Camera3RequestDescriptor::Status::Error); + processingStatus = Camera3RequestDescriptor::Status::Error; } } + if (needsPostProcessing) { + if (processingStatus == Camera3RequestDescriptor::Status::Error) { + descriptor->status_ = processingStatus; + sendCaptureResults(); + } + + return; + } + descriptor->status_ = Camera3RequestDescriptor::Status::Success; sendCaptureResults(); } @@ -1206,6 +1220,64 @@ void CameraDevice::sendCaptureResults() } } +void CameraDevice::setBufferStatus(CameraStream *cameraStream, + Camera3RequestDescriptor::StreamBuffer &buffer, + Camera3RequestDescriptor *request, + Camera3RequestDescriptor::Status status) +{ + /* + * Return the FrameBuffer to the CameraStream now that we're + * done processing it. + */ + if (cameraStream->type() == CameraStream::Type::Internal) + cameraStream->putBuffer(buffer.internalBuffer); + + buffer.status = status; + if (status != Camera3RequestDescriptor::Status::Success) + notifyError(request->frameNumber_, buffer.stream->camera3Stream(), + CAMERA3_MSG_ERROR_BUFFER); +} + +void CameraDevice::streamProcessingComplete(CameraStream *cameraStream, + Camera3RequestDescriptor *request, + Camera3RequestDescriptor::Status status) +{ + MutexLocker locker(request->streamsProcessMutex_); + for (auto &buffer : request->buffers_) { + if (buffer.stream != cameraStream) + continue; + + setBufferStatus(cameraStream, buffer, request, status); + } + + bool hasPostProcessingErrors = false; + for (auto &buffer : request->buffers_) { + if (cameraStream->type() == CameraStream::Type::Direct) + continue; + + /* + * Other eligible buffers might be waiting to get post-processed. + * So wait for their turn before sendCaptureResults() for the + * descriptor. + */ + if (buffer.status == Camera3RequestDescriptor::Status::Pending) + return; + + if (!hasPostProcessingErrors && + buffer.status == Camera3RequestDescriptor::Status::Error) + hasPostProcessingErrors = true; + } + + if (hasPostProcessingErrors) + request->status_ = Camera3RequestDescriptor::Status::Error; + else + request->status_ = Camera3RequestDescriptor::Status::Success; + + locker.unlock(); + + sendCaptureResults(); +} + std::string CameraDevice::logPrefix() const { return "'" + camera_->id() + "'"; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index 863cf414..1ef933da 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -66,6 +66,9 @@ public: int configureStreams(camera3_stream_configuration_t *stream_list); int processCaptureRequest(camera3_capture_request_t *request); void requestComplete(libcamera::Request *request); + void streamProcessingComplete(CameraStream *cameraStream, + Camera3RequestDescriptor *request, + Camera3RequestDescriptor::Status status); protected: std::string logPrefix() const override; @@ -94,6 +97,10 @@ private: camera3_error_msg_code code) const; int processControls(Camera3RequestDescriptor *descriptor); void sendCaptureResults(); + void setBufferStatus(CameraStream *cameraStream, + Camera3RequestDescriptor::StreamBuffer &buffer, + Camera3RequestDescriptor *request, + Camera3RequestDescriptor::Status status); std::unique_ptr getResultMetadata( const Camera3RequestDescriptor &descriptor) const; diff --git a/src/android/camera_request.h b/src/android/camera_request.h index 05dabf89..3a2774e0 100644 --- a/src/android/camera_request.h +++ b/src/android/camera_request.h @@ -8,6 +8,7 @@ #define __ANDROID_CAMERA_REQUEST_H__ #include +#include #include #include @@ -37,6 +38,7 @@ public: std::unique_ptr frameBuffer; int fence; Status status; + libcamera::FrameBuffer *internalBuffer = nullptr; }; Camera3RequestDescriptor(libcamera::Camera *camera, @@ -47,6 +49,8 @@ public: uint32_t frameNumber_ = 0; + /* Protects buffers_ for post-processing streams. */ + std::mutex streamsProcessMutex_; std::vector buffers_; CameraMetadata settings_; diff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp index f44a2717..04cbef8c 100644 --- a/src/android/camera_stream.cpp +++ b/src/android/camera_stream.cpp @@ -22,6 +22,7 @@ #include "camera_capabilities.h" #include "camera_device.h" #include "camera_metadata.h" +#include "post_processor.h" using namespace libcamera; @@ -97,6 +98,18 @@ int CameraStream::configure() int ret = postProcessor_->configure(configuration(), output); if (ret) return ret; + + postProcessor_->processComplete.connect( + this, [&](Camera3RequestDescriptor *request, PostProcessor::Status status) { + Camera3RequestDescriptor::Status bufferStatus = + Camera3RequestDescriptor::Status::Error; + + if (status == PostProcessor::Status::Success) + bufferStatus = Camera3RequestDescriptor::Status::Success; + + cameraDevice_->streamProcessingComplete(this, request, + bufferStatus); + }); } if (type_ == Type::Internal) { diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp index 699576ef..a001fede 100644 --- a/src/android/jpeg/post_processor_jpeg.cpp +++ b/src/android/jpeg/post_processor_jpeg.cpp @@ -198,6 +198,7 @@ int PostProcessorJpeg::process(const FrameBuffer &source, exif.data(), quality); if (jpeg_size < 0) { LOG(JPEG, Error) << "Failed to encode stream image"; + processComplete.emit(request, PostProcessor::Status::Error); return jpeg_size; } @@ -211,6 +212,7 @@ int PostProcessorJpeg::process(const FrameBuffer &source, /* Update the JPEG result Metadata. */ resultMetadata->addEntry(ANDROID_JPEG_SIZE, jpeg_size); + processComplete.emit(request, PostProcessor::Status::Success); return 0; } diff --git a/src/android/post_processor.h b/src/android/post_processor.h index 27eaef88..14f5e8c7 100644 --- a/src/android/post_processor.h +++ b/src/android/post_processor.h @@ -7,6 +7,8 @@ #ifndef __ANDROID_POST_PROCESSOR_H__ #define __ANDROID_POST_PROCESSOR_H__ +#include + #include #include @@ -17,6 +19,11 @@ class Camera3RequestDescriptor; class PostProcessor { public: + enum class Status { + Error, + Success + }; + virtual ~PostProcessor() = default; virtual int configure(const libcamera::StreamConfiguration &inCfg, @@ -24,6 +31,8 @@ public: virtual int process(const libcamera::FrameBuffer &source, CameraBuffer *destination, Camera3RequestDescriptor *request) = 0; + + libcamera::Signal processComplete; }; #endif /* __ANDROID_POST_PROCESSOR_H__ */ diff --git a/src/android/yuv/post_processor_yuv.cpp b/src/android/yuv/post_processor_yuv.cpp index 8110a1f1..fd364741 100644 --- a/src/android/yuv/post_processor_yuv.cpp +++ b/src/android/yuv/post_processor_yuv.cpp @@ -51,14 +51,17 @@ int PostProcessorYuv::configure(const StreamConfiguration &inCfg, int PostProcessorYuv::process(const FrameBuffer &source, CameraBuffer *destination, - [[maybe_unused]] Camera3RequestDescriptor *request) + Camera3RequestDescriptor *request) { - if (!isValidBuffers(source, *destination)) + if (!isValidBuffers(source, *destination)) { + processComplete.emit(request, PostProcessor::Status::Error); return -EINVAL; + } const MappedFrameBuffer sourceMapped(&source, MappedFrameBuffer::MapFlag::Read); if (!sourceMapped.isValid()) { LOG(YUV, Error) << "Failed to mmap camera frame buffer"; + processComplete.emit(request, PostProcessor::Status::Error); return -EINVAL; } @@ -76,9 +79,12 @@ int PostProcessorYuv::process(const FrameBuffer &source, libyuv::FilterMode::kFilterBilinear); if (ret) { LOG(YUV, Error) << "Failed NV12 scaling: " << ret; + processComplete.emit(request, PostProcessor::Status::Error); return -EINVAL; } + processComplete.emit(request, PostProcessor::Status::Success); + return 0; }