From patchwork Thu Mar 26 16:08:13 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3336 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id DF98060414 for ; Thu, 26 Mar 2020 17:10:32 +0100 (CET) X-Halon-ID: 4c290c59-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4c290c59-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:23 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:13 +0100 Message-Id: <20200326160819.4088361-2-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 1/7] ipa: Add start() and stop() operations X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:33 -0000 Add two new operations to the IPA interface to start and stop it. The intention is that these functions shall be used by the IPA to perform in actions in conjunction with the camera start and stop. Signed-off-by: Niklas Söderlund --- include/ipa/ipa_interface.h | 4 ++++ include/ipa/ipa_vimc.h | 2 ++ src/ipa/libipa/ipa_interface_wrapper.cpp | 16 +++++++++++++ src/ipa/libipa/ipa_interface_wrapper.h | 2 ++ src/ipa/rkisp1/rkisp1.cpp | 2 ++ src/ipa/vimc/vimc.cpp | 20 ++++++++++++++++ src/libcamera/include/ipa_context_wrapper.h | 2 ++ src/libcamera/ipa_context_wrapper.cpp | 26 +++++++++++++++++++++ src/libcamera/proxy/ipa_proxy_linux.cpp | 2 ++ test/ipa/ipa_interface_test.cpp | 24 +++++++++++++++++++ test/ipa/ipa_wrappers_test.cpp | 25 ++++++++++++++++++-- 11 files changed, 123 insertions(+), 2 deletions(-) diff --git a/include/ipa/ipa_interface.h b/include/ipa/ipa_interface.h index 229d1124b19ec1c9..9a62208cc80084c8 100644 --- a/include/ipa/ipa_interface.h +++ b/include/ipa/ipa_interface.h @@ -64,6 +64,8 @@ struct ipa_context_ops { void (*destroy)(struct ipa_context *ctx); void *(*get_interface)(struct ipa_context *ctx); void (*init)(struct ipa_context *ctx); + void (*start)(struct ipa_context *ctx); + void (*stop)(struct ipa_context *ctx); void (*register_callbacks)(struct ipa_context *ctx, const struct ipa_callback_ops *callbacks, void *cb_ctx); @@ -120,6 +122,8 @@ public: virtual ~IPAInterface() {} virtual int init() = 0; + virtual int start() = 0; + virtual void stop() = 0; virtual void configure(const std::map &streamConfig, const std::map &entityControls) = 0; diff --git a/include/ipa/ipa_vimc.h b/include/ipa/ipa_vimc.h index 9add122cf598aa6f..8e82dd94bf479e35 100644 --- a/include/ipa/ipa_vimc.h +++ b/include/ipa/ipa_vimc.h @@ -15,6 +15,8 @@ namespace libcamera { enum IPAOperationCode { IPAOperationNone, IPAOperationInit, + IPAOperationStart, + IPAOperationStop, }; } /* namespace libcamera */ diff --git a/src/ipa/libipa/ipa_interface_wrapper.cpp b/src/ipa/libipa/ipa_interface_wrapper.cpp index b93c1c1f1c1a2018..ef074ba17316ed97 100644 --- a/src/ipa/libipa/ipa_interface_wrapper.cpp +++ b/src/ipa/libipa/ipa_interface_wrapper.cpp @@ -86,6 +86,20 @@ void IPAInterfaceWrapper::init(struct ipa_context *_ctx) ctx->ipa_->init(); } +void IPAInterfaceWrapper::start(struct ipa_context *_ctx) +{ + IPAInterfaceWrapper *ctx = static_cast(_ctx); + + ctx->ipa_->start(); +} + +void IPAInterfaceWrapper::stop(struct ipa_context *_ctx) +{ + IPAInterfaceWrapper *ctx = static_cast(_ctx); + + ctx->ipa_->stop(); +} + void IPAInterfaceWrapper::register_callbacks(struct ipa_context *_ctx, const struct ipa_callback_ops *callbacks, void *cb_ctx) @@ -234,6 +248,8 @@ const struct ipa_context_ops IPAInterfaceWrapper::operations_ = { .destroy = &IPAInterfaceWrapper::destroy, .get_interface = &IPAInterfaceWrapper::get_interface, .init = &IPAInterfaceWrapper::init, + .start = &IPAInterfaceWrapper::start, + .stop = &IPAInterfaceWrapper::stop, .register_callbacks = &IPAInterfaceWrapper::register_callbacks, .configure = &IPAInterfaceWrapper::configure, .map_buffers = &IPAInterfaceWrapper::map_buffers, diff --git a/src/ipa/libipa/ipa_interface_wrapper.h b/src/ipa/libipa/ipa_interface_wrapper.h index 3fb7b447643953ff..9ee4fe15b2738cf4 100644 --- a/src/ipa/libipa/ipa_interface_wrapper.h +++ b/src/ipa/libipa/ipa_interface_wrapper.h @@ -24,6 +24,8 @@ private: static void destroy(struct ipa_context *ctx); static void *get_interface(struct ipa_context *ctx); static void init(struct ipa_context *ctx); + static void start(struct ipa_context *ctx); + static void stop(struct ipa_context *ctx); static void register_callbacks(struct ipa_context *ctx, const struct ipa_callback_ops *callbacks, void *cb_ctx); diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 438b3c66f77a1e57..af38e3299f2ff045 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -33,6 +33,8 @@ class IPARkISP1 : public IPAInterface { public: int init() override { return 0; } + int start() override { return 0; } + void stop() override {} void configure(const std::map &streamConfig, const std::map &entityControls) override; diff --git a/src/ipa/vimc/vimc.cpp b/src/ipa/vimc/vimc.cpp index 6e2095b56bbcdaa3..566a66e404affbd5 100644 --- a/src/ipa/vimc/vimc.cpp +++ b/src/ipa/vimc/vimc.cpp @@ -32,6 +32,10 @@ public: ~IPAVimc(); int init() override; + + int start() override; + void stop() override; + void configure(const std::map &streamConfig, const std::map &entityControls) override {} void mapBuffers(const std::vector &buffers) override {} @@ -66,6 +70,22 @@ int IPAVimc::init() return 0; } +int IPAVimc::start() +{ + trace(IPAOperationStart); + + LOG(IPAVimc, Debug) << "start vimc IPA!"; + + return 0; +} + +void IPAVimc::stop() +{ + trace(IPAOperationStop); + + LOG(IPAVimc, Debug) << "stop vimc IPA!"; +} + void IPAVimc::initTrace() { struct stat fifoStat; diff --git a/src/libcamera/include/ipa_context_wrapper.h b/src/libcamera/include/ipa_context_wrapper.h index c9e194120de6b69c..0a48bfe732c831d7 100644 --- a/src/libcamera/include/ipa_context_wrapper.h +++ b/src/libcamera/include/ipa_context_wrapper.h @@ -20,6 +20,8 @@ public: ~IPAContextWrapper(); int init() override; + int start() override; + void stop() override; void configure(const std::map &streamConfig, const std::map &entityControls) override; diff --git a/src/libcamera/ipa_context_wrapper.cpp b/src/libcamera/ipa_context_wrapper.cpp index 946a2fd8cab1782a..c40e56169512a1ba 100644 --- a/src/libcamera/ipa_context_wrapper.cpp +++ b/src/libcamera/ipa_context_wrapper.cpp @@ -82,6 +82,32 @@ int IPAContextWrapper::init() return 0; } +int IPAContextWrapper::start() +{ + if (intf_) + return intf_->start(); + + if (!ctx_) + return 0; + + ctx_->ops->start(ctx_); + + return 0; +} + +void IPAContextWrapper::stop() +{ + if (intf_) + return intf_->stop(); + + if (!ctx_) + return; + + ctx_->ops->stop(ctx_); + + return; +} + void IPAContextWrapper::configure(const std::map &streamConfig, const std::map &entityControls) { diff --git a/src/libcamera/proxy/ipa_proxy_linux.cpp b/src/libcamera/proxy/ipa_proxy_linux.cpp index c7218fb472490cc8..2aa80b9467044fbf 100644 --- a/src/libcamera/proxy/ipa_proxy_linux.cpp +++ b/src/libcamera/proxy/ipa_proxy_linux.cpp @@ -27,6 +27,8 @@ public: ~IPAProxyLinux(); int init() override { return 0; } + int start() override { return 0; } + void stop() override {} void configure(const std::map &streamConfig, const std::map &entityControls) override {} void mapBuffers(const std::vector &buffers) override {} diff --git a/test/ipa/ipa_interface_test.cpp b/test/ipa/ipa_interface_test.cpp index cafc249bbbc96cf8..f861c10f16925d78 100644 --- a/test/ipa/ipa_interface_test.cpp +++ b/test/ipa/ipa_interface_test.cpp @@ -109,6 +109,30 @@ protected: return TestFail; } + /* Test start of IPA module. */ + ipa_->start(); + timer.start(1000); + while (timer.isRunning() && trace_ != IPAOperationStart) + dispatcher->processEvents(); + + if (trace_ != IPAOperationStart) { + cerr << "Failed to test IPA start sequence" + << endl; + return TestFail; + } + + /* Test start of IPA module. */ + ipa_->stop(); + timer.start(1000); + while (timer.isRunning() && trace_ != IPAOperationStop) + dispatcher->processEvents(); + + if (trace_ != IPAOperationStop) { + cerr << "Failed to test IPA stop sequence" + << endl; + return TestFail; + } + return TestPass; } diff --git a/test/ipa/ipa_wrappers_test.cpp b/test/ipa/ipa_wrappers_test.cpp index 1ae1781169c85e6c..ac56498c40e597c9 100644 --- a/test/ipa/ipa_wrappers_test.cpp +++ b/test/ipa/ipa_wrappers_test.cpp @@ -27,6 +27,8 @@ using namespace std; enum Operation { Op_init, + Op_start, + Op_stop, Op_configure, Op_mapBuffers, Op_unmapBuffers, @@ -47,6 +49,17 @@ public: return 0; } + int start() override + { + report(Op_start, TestPass); + return 0; + } + + void stop() override + { + report(Op_stop, TestPass); + } + void configure(const std::map &streamConfig, const std::map &entityControls) override { @@ -323,13 +336,21 @@ protected: return TestFail; /* - * Test init() last to ensure nothing in the wrappers or - * serializer depends on init() being called first. + * Test init(), start() and stop() last to ensure nothing in the + * wrappers or serializer depends on them being called first. */ ret = INVOKE(init); if (ret == TestFail) return TestFail; + ret = INVOKE(start); + if (ret == TestFail) + return TestFail; + + ret = INVOKE(stop); + if (ret == TestFail) + return TestFail; + return TestPass; } From patchwork Thu Mar 26 16:08:14 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3337 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 32E5E62684 for ; Thu, 26 Mar 2020 17:10:33 +0100 (CET) X-Halon-ID: 4cb2ccaf-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4cb2ccaf-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:24 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:14 +0100 Message-Id: <20200326160819.4088361-3-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 2/7] libcamera: pipeline: rkisp1: Queue parameters even if they are not ready X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:33 -0000 If the IPA have not filled in the parameters buffer still queue it to hardware. Not queuing the buffer results in the pipeline and hardware going out of sync. This is not a permanent fix of the problem and a todo is added to fix it properly. This change do not make the situation worse as the state of the pipeline is just as unknown as if no param buffer is queued as if one with old content in it. Signed-off-by: Niklas Söderlund Acked-by: Laurent Pinchart --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index 2f909cef7c75ba0f..4ec74c5aa8286ffb 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -351,13 +351,23 @@ protected: if (!info) LOG(RkISP1, Fatal) << "Frame not known"; - if (info->paramFilled) - pipe_->param_->queueBuffer(info->paramBuffer); - else + /* + * \todo: If parameters are not filled a better method to handle + * the situation then queing a buffer with unkown content should + * be used. + * + * It seems obsessive to keep an internal zeroed scratch + * parameters buffer around as this should not happen uless the + * devices is under too much load. Perhaps failing the request + * and returning it to the applicaiton with an error code is better + * then to queue it to hardware? + */ + if (!info->paramFilled) LOG(RkISP1, Error) << "Parameters not ready on time for frame " - << frame() << ", ignore parameters."; + << frame(); + pipe_->param_->queueBuffer(info->paramBuffer); pipe_->stat_->queueBuffer(info->statBuffer); pipe_->video_->queueBuffer(info->videoBuffer); } From patchwork Thu Mar 26 16:08:15 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3338 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id C2A8F60414 for ; Thu, 26 Mar 2020 17:10:33 +0100 (CET) X-Halon-ID: 4cfa37a5-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4cfa37a5-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:24 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:15 +0100 Message-Id: <20200326160819.4088361-4-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 3/7] libcamera: pipeline: rkisp1: Initialize IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:34 -0000 The IPA init() call is a no-op in the current IPA, but it's part of the IPA API and it should be called. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index 4ec74c5aa8286ffb..f64807984519779b 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -386,6 +386,8 @@ int RkISP1CameraData::loadIPA() ipa_->queueFrameAction.connect(this, &RkISP1CameraData::queueFrameAction); + ipa_->init(); + return 0; } From patchwork Thu Mar 26 16:08:16 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3339 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1836762CEB for ; Thu, 26 Mar 2020 17:10:34 +0100 (CET) X-Halon-ID: 4d3d0928-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4d3d0928-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:25 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:16 +0100 Message-Id: <20200326160819.4088361-5-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 4/7] libcamera: pipeline: rkisp1: Do not try to process cancelled frames X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:36 -0000 Their is no need to try and process cancelled frames. Signed-off-by: Niklas Söderlund --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index f64807984519779b..f49b3a7f0945ac92 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -1011,6 +1011,9 @@ void PipelineHandlerRkISP1::tryCompleteRequest(Request *request) void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) { + if (buffer->metadata().status == FrameMetadata::FrameCancelled) + return; + ASSERT(activeCamera_); RkISP1CameraData *data = cameraData(activeCamera_); Request *request = buffer->request(); @@ -1026,6 +1029,9 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) void PipelineHandlerRkISP1::paramReady(FrameBuffer *buffer) { + if (buffer->metadata().status == FrameMetadata::FrameCancelled) + return; + ASSERT(activeCamera_); RkISP1CameraData *data = cameraData(activeCamera_); @@ -1037,6 +1043,9 @@ void PipelineHandlerRkISP1::paramReady(FrameBuffer *buffer) void PipelineHandlerRkISP1::statReady(FrameBuffer *buffer) { + if (buffer->metadata().status == FrameMetadata::FrameCancelled) + return; + ASSERT(activeCamera_); RkISP1CameraData *data = cameraData(activeCamera_); From patchwork Thu Mar 26 16:08:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3340 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id AC26962D29 for ; Thu, 26 Mar 2020 17:10:34 +0100 (CET) X-Halon-ID: 4d7f1db1-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4d7f1db1-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:25 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:17 +0100 Message-Id: <20200326160819.4088361-6-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 5/7] libcamera: pipeline: rkisp1: Call IPA start() and stop() X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:36 -0000 Call the IPA start()/stop() functions before/after the camera is started. This makes sure the IPA functions properly once thread management is moved into the start/stop interface. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index f49b3a7f0945ac92..3287fd3a18204cbc 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -113,8 +113,8 @@ class RkISP1CameraData : public CameraData { public: RkISP1CameraData(PipelineHandler *pipe) - : CameraData(pipe), sensor_(nullptr), frame_(0), - frameInfo_(pipe) + : CameraData(pipe), running_(false), sensor_(nullptr), + frame_(0), frameInfo_(pipe) { } @@ -125,6 +125,7 @@ public: int loadIPA(); + bool running_; Stream stream_; CameraSensor *sensor_; unsigned int frame_; @@ -394,6 +395,9 @@ int RkISP1CameraData::loadIPA() void RkISP1CameraData::queueFrameAction(unsigned int frame, const IPAOperationData &action) { + if (!running_) + return; + switch (action.operation) { case RKISP1_IPA_ACTION_V4L2_SET: { const ControlList &controls = action.controls[0]; @@ -764,10 +768,19 @@ int PipelineHandlerRkISP1::start(Camera *camera) if (ret) return ret; + ret = data->ipa_->start(); + if (ret) { + freeBuffers(camera); + LOG(RkISP1, Error) + << "Failed to start IPA " << camera->name(); + return ret; + } + data->frame_ = 0; ret = param_->streamOn(); if (ret) { + data->ipa_->stop(); freeBuffers(camera); LOG(RkISP1, Error) << "Failed to start parameters " << camera->name(); @@ -777,6 +790,7 @@ int PipelineHandlerRkISP1::start(Camera *camera) ret = stat_->streamOn(); if (ret) { param_->streamOff(); + data->ipa_->stop(); freeBuffers(camera); LOG(RkISP1, Error) << "Failed to start statistics " << camera->name(); @@ -787,12 +801,14 @@ int PipelineHandlerRkISP1::start(Camera *camera) if (ret) { param_->streamOff(); stat_->streamOff(); + data->ipa_->stop(); freeBuffers(camera); LOG(RkISP1, Error) << "Failed to start camera " << camera->name(); } + data->running_ = true; activeCamera_ = camera; /* Inform IPA of stream configuration and sensor controls. */ @@ -830,6 +846,10 @@ void PipelineHandlerRkISP1::stop(Camera *camera) LOG(RkISP1, Warning) << "Failed to stop parameters " << camera->name(); + data->running_ = false; + + data->ipa_->stop(); + data->timeline_.reset(); freeBuffers(camera); From patchwork Thu Mar 26 16:08:18 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3341 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 156FC62693 for ; Thu, 26 Mar 2020 17:10:35 +0100 (CET) X-Halon-ID: 4dcbd111-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4dcbd111-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:26 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:18 +0100 Message-Id: <20200326160819.4088361-7-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 6/7] libcamera: pipeline: vimc: Call IPA start() and stop() X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:36 -0000 Call the IPA start()/stop() functions before/after the camera is started. This makes sure the IPA functions properly once thread management is moved into the start/stop interface. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/pipeline/vimc/vimc.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libcamera/pipeline/vimc/vimc.cpp b/src/libcamera/pipeline/vimc/vimc.cpp index b04a9726efa5ade6..c5eea3a01b0e9446 100644 --- a/src/libcamera/pipeline/vimc/vimc.cpp +++ b/src/libcamera/pipeline/vimc/vimc.cpp @@ -274,8 +274,15 @@ int PipelineHandlerVimc::start(Camera *camera) if (ret < 0) return ret; + ret = data->ipa_->start(); + if (ret) { + data->video_->releaseBuffers(); + return ret; + } + ret = data->video_->streamOn(); if (ret < 0) { + data->ipa_->stop(); data->video_->releaseBuffers(); return ret; } @@ -287,6 +294,7 @@ void PipelineHandlerVimc::stop(Camera *camera) { VimcCameraData *data = cameraData(camera); data->video_->streamOff(); + data->ipa_->stop(); data->video_->releaseBuffers(); } From patchwork Thu Mar 26 16:08:19 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 3342 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id E349E605D2 for ; Thu, 26 Mar 2020 17:10:35 +0100 (CET) X-Halon-ID: 4e0f3aa4-6f7c-11ea-89d0-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (p4fca2392.dip0.t-ipconnect.de [79.202.35.146]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 4e0f3aa4-6f7c-11ea-89d0-0050569116f7; Thu, 26 Mar 2020 17:10:26 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 26 Mar 2020 17:08:19 +0100 Message-Id: <20200326160819.4088361-8-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> References: <20200326160819.4088361-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy open-source IPAs to a thread X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Mar 2020 16:10:37 -0000 From: Laurent Pinchart While closed-source IPA modules will always be sandboxed, open-source IPA modules may be run in the main libcamera process or be sandboxed, depending on platform configuration. These two models exhibit very different timings, which require extensive testing with both configurations. When run into the main libcamera process, IPA modules are executed in the pipeline handler thread (which is currently a global CameraManager thread). Time-consuming operations in the IPA may thus slow down the pipeline handler and compromise real-time behaviour. At least some pipeline handlers will thus likely spawn a thread to isolate the IPA, leading to code duplication in pipeline handlers. Solve both issues by always proxying IPA modules. For open-source IPA modules that run in the libcamera process, a new IPAProxyThread class is added to run the IPA in a separate thread. Signed-off-by: Laurent Pinchart [Niklas: Move thread start/stop of thread into start()/stop()] Signed-off-by: Niklas Söderlund --- src/libcamera/ipa_manager.cpp | 48 ++++---- src/libcamera/proxy/ipa_proxy_thread.cpp | 144 +++++++++++++++++++++++ src/libcamera/proxy/meson.build | 1 + 3 files changed, 166 insertions(+), 27 deletions(-) create mode 100644 src/libcamera/proxy/ipa_proxy_thread.cpp diff --git a/src/libcamera/ipa_manager.cpp b/src/libcamera/ipa_manager.cpp index bcaae3564ea14e07..6d23f470506561b8 100644 --- a/src/libcamera/ipa_manager.cpp +++ b/src/libcamera/ipa_manager.cpp @@ -12,7 +12,6 @@ #include #include -#include "ipa_context_wrapper.h" #include "ipa_module.h" #include "ipa_proxy.h" #include "log.h" @@ -271,40 +270,35 @@ std::unique_ptr IPAManager::createIPA(PipelineHandler *pipe, if (!m) return nullptr; - if (!m->isOpenSource()) { - IPAProxyFactory *pf = nullptr; - std::vector &factories = IPAProxyFactory::factories(); + /* + * Load and run the IPA module in a thread if it is open-source, or + * isolate it in a separate process otherwise. + * + * \todo Implement a better proxy selection + */ + const char *proxyName = m->isOpenSource() + ? "IPAProxyThread" : "IPAProxyLinux"; + IPAProxyFactory *pf = nullptr; - for (IPAProxyFactory *factory : factories) { - /* TODO: Better matching */ - if (!strcmp(factory->name().c_str(), "IPAProxyLinux")) { - pf = factory; - break; - } + for (IPAProxyFactory *factory : IPAProxyFactory::factories()) { + if (!strcmp(factory->name().c_str(), proxyName)) { + pf = factory; + break; } - - if (!pf) { - LOG(IPAManager, Error) << "Failed to get proxy factory"; - return nullptr; - } - - std::unique_ptr proxy = pf->create(m); - if (!proxy->isValid()) { - LOG(IPAManager, Error) << "Failed to load proxy"; - return nullptr; - } - - return proxy; } - if (!m->load()) + if (!pf) { + LOG(IPAManager, Error) << "Failed to get proxy factory"; return nullptr; + } - struct ipa_context *ctx = m->createContext(); - if (!ctx) + std::unique_ptr proxy = pf->create(m); + if (!proxy->isValid()) { + LOG(IPAManager, Error) << "Failed to load proxy"; return nullptr; + } - return std::make_unique(ctx); + return proxy; } } /* namespace libcamera */ diff --git a/src/libcamera/proxy/ipa_proxy_thread.cpp b/src/libcamera/proxy/ipa_proxy_thread.cpp new file mode 100644 index 0000000000000000..6719bcdda43f3647 --- /dev/null +++ b/src/libcamera/proxy/ipa_proxy_thread.cpp @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * ipa_proxy_thread.cpp - Proxy running an Image Processing Algorithm in a thread + */ + +#include + +#include +#include + +#include "ipa_context_wrapper.h" +#include "ipa_module.h" +#include "ipa_proxy.h" +#include "log.h" +#include "thread.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(IPAProxy) + +class IPAProxyThread : public IPAProxy, public Object +{ +public: + IPAProxyThread(IPAModule *ipam); + + int init() override; + int start() override; + void stop() override; + + void configure(const std::map &streamConfig, + const std::map &entityControls) override; + void mapBuffers(const std::vector &buffers) override; + void unmapBuffers(const std::vector &ids) override; + void processEvent(const IPAOperationData &event) override; + +private: + void queueFrameAction(unsigned int frame, const IPAOperationData &data); + + /* Helper class to invoke processEvent() in another thread. */ + class ThreadProxy : public Object + { + public: + void setIPA(IPAInterface *ipa) + { + ipa_ = ipa; + } + + void stop() + { + ipa_->stop(); + } + + void processEvent(const IPAOperationData &event) + { + ipa_->processEvent(event); + } + + private: + IPAInterface *ipa_; + }; + + Thread thread_; + ThreadProxy proxy_; + std::unique_ptr ipa_; +}; + +IPAProxyThread::IPAProxyThread(IPAModule *ipam) +{ + if (!ipam->load()) + return; + + struct ipa_context *ctx = ipam->createContext(); + if (!ctx) { + LOG(IPAProxy, Error) + << "Failed to create IPA context for " << ipam->path(); + return; + } + + ipa_ = std::make_unique(ctx); + proxy_.setIPA(ipa_.get()); + + /* + * Proxy the queueFrameAction signal to dispatch it in the caller's + * thread. + */ + ipa_->queueFrameAction.connect(this, &IPAProxyThread::queueFrameAction); + + valid_ = true; +} + +int IPAProxyThread::init() +{ + return ipa_->init(); +} + +int IPAProxyThread::start() +{ + thread_.start(); + proxy_.moveToThread(&thread_); + + return ipa_->start(); +} + +void IPAProxyThread::stop() +{ + proxy_.invokeMethod(&ThreadProxy::stop, ConnectionTypeBlocking); + + thread_.exit(); + thread_.wait(); +} + +void IPAProxyThread::configure(const std::map &streamConfig, + const std::map &entityControls) +{ + ipa_->configure(streamConfig, entityControls); +} + +void IPAProxyThread::mapBuffers(const std::vector &buffers) +{ + ipa_->mapBuffers(buffers); +} + +void IPAProxyThread::unmapBuffers(const std::vector &ids) +{ + ipa_->unmapBuffers(ids); +} + +void IPAProxyThread::processEvent(const IPAOperationData &event) +{ + /* Dispatch the processEvent() call to the thread. */ + proxy_.invokeMethod(&ThreadProxy::processEvent, ConnectionTypeQueued, + event); +} + +void IPAProxyThread::queueFrameAction(unsigned int frame, const IPAOperationData &data) +{ + IPAInterface::queueFrameAction.emit(frame, data); +} + +REGISTER_IPA_PROXY(IPAProxyThread) + +} /* namespace libcamera */ diff --git a/src/libcamera/proxy/meson.build b/src/libcamera/proxy/meson.build index efc1132302176f63..6c00d5f30ad2e5f0 100644 --- a/src/libcamera/proxy/meson.build +++ b/src/libcamera/proxy/meson.build @@ -1,3 +1,4 @@ libcamera_sources += files([ 'ipa_proxy_linux.cpp', + 'ipa_proxy_thread.cpp', ])