From patchwork Tue Oct 8 00:45:25 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2139 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7253161966 for ; Tue, 8 Oct 2019 02:46:41 +0200 (CEST) X-Halon-ID: f35c9035-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f35c9035-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:40 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:25 +0200 Message-Id: <20191008004534.1585142-2-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 01/10] libcamera: pipeline: Move IPA from pipeline to camera data X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:41 -0000 The IPA acts on a camera and not on a pipeline which can expose more then one camera. Move the IPA reference to the CameraData. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/include/pipeline_handler.h | 2 ++ src/libcamera/pipeline/vimc.cpp | 12 +++++------- src/libcamera/pipeline_handler.cpp | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/libcamera/include/pipeline_handler.h b/src/libcamera/include/pipeline_handler.h index 1fdef9cea40f1f0a..42b90a4bf1a6e414 100644 --- a/src/libcamera/include/pipeline_handler.h +++ b/src/libcamera/include/pipeline_handler.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ public: PipelineHandler *pipe_; std::list queuedRequests_; ControlInfoMap controlInfo_; + std::unique_ptr ipa_; private: CameraData(const CameraData &) = delete; diff --git a/src/libcamera/pipeline/vimc.cpp b/src/libcamera/pipeline/vimc.cpp index 26477dccbb8fcd5b..f1cfd0ed35cfd9cd 100644 --- a/src/libcamera/pipeline/vimc.cpp +++ b/src/libcamera/pipeline/vimc.cpp @@ -101,8 +101,6 @@ private: return static_cast( PipelineHandler::cameraData(camera)); } - - std::unique_ptr ipa_; }; VimcCameraConfiguration::VimcCameraConfiguration() @@ -353,13 +351,13 @@ bool PipelineHandlerVimc::match(DeviceEnumerator *enumerator) if (!media) return false; - ipa_ = IPAManager::instance()->createIPA(this, 0, 0); - if (ipa_ == nullptr) + std::unique_ptr data = utils::make_unique(this); + + data->ipa_ = IPAManager::instance()->createIPA(this, 0, 0); + if (data->ipa_ == nullptr) LOG(VIMC, Warning) << "no matching IPA found"; else - ipa_->init(); - - std::unique_ptr data = utils::make_unique(this); + data->ipa_->init(); /* Locate and open the capture video node. */ if (data->init(media)) diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index 3e54aa23d92b9a36..cb4337e50eb6c268 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -96,6 +96,14 @@ LOG_DEFINE_CATEGORY(Pipeline) * creating the camera, and shall not be modified afterwards. */ +/** + * \var CameraData::ipa_ + * \brief The IPA module used by the camera + * + * Reference to the Image Processing Algorithms (IPA) operating on the camera's + * stream(s). If no IPA exists for the camera, this field is set to nullptr. + */ + /** * \class PipelineHandler * \brief Create and manage cameras based on a set of media devices From patchwork Tue Oct 8 00:45:26 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2140 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 726C961968 for ; Tue, 8 Oct 2019 02:46:42 +0200 (CEST) X-Halon-ID: f3efde37-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f3efde37-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:41 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:26 +0200 Message-Id: <20191008004534.1585142-3-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 02/10] libcamera: controls: Fix documentation of AwbEnable X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:42 -0000 Align the wording of the documentation for the AwbEnable control with review comments for similar controls. Suggested-by: Laurent Pinchart Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/control_ids.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/control_ids.yaml b/src/libcamera/control_ids.yaml index 819a5967a2fc8d56..0e18ec9397cdaa05 100644 --- a/src/libcamera/control_ids.yaml +++ b/src/libcamera/control_ids.yaml @@ -8,7 +8,7 @@ controls: - AwbEnable: type: bool description: | - Enables or disables the AWB. + Enable or disable the AWB. \sa ManualGain From patchwork Tue Oct 8 00:45:27 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2141 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 0A53360E1F for ; Tue, 8 Oct 2019 02:46:43 +0200 (CEST) X-Halon-ID: f46d0a50-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f46d0a50-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:42 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:27 +0200 Message-Id: <20191008004534.1585142-4-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 03/10] libcamera: controls: Add AeEnable control X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:43 -0000 Add a control to enable/disable Auto Exposure. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/control_ids.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcamera/control_ids.yaml b/src/libcamera/control_ids.yaml index 0e18ec9397cdaa05..1164cb6a40dedc27 100644 --- a/src/libcamera/control_ids.yaml +++ b/src/libcamera/control_ids.yaml @@ -5,6 +5,13 @@ %YAML 1.2 --- controls: + - AeEnable: + type: bool + description: | + Enable or disable the AE. + + \sa ManualExposure + - AwbEnable: type: bool description: | From patchwork Tue Oct 8 00:45:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2142 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D263761968 for ; Tue, 8 Oct 2019 02:46:43 +0200 (CEST) X-Halon-ID: f4e51b66-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f4e51b66-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:43 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:28 +0200 Message-Id: <20191008004534.1585142-5-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 04/10] libcamera: controls: Add AeLocked control X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:44 -0000 Add a control to report lock status of the Auto Exposure algorithm. The idea is that if an AE algorithm is running the control shall be added to the metadata control list. If the AE algorithm is locked the value shall be set to true, if it's converging it shall be set to false. If the AE algorithm is not running the control shall not be present in the metadata control list. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/control_ids.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libcamera/control_ids.yaml b/src/libcamera/control_ids.yaml index 1164cb6a40dedc27..4befec746a59e1c3 100644 --- a/src/libcamera/control_ids.yaml +++ b/src/libcamera/control_ids.yaml @@ -12,6 +12,17 @@ controls: \sa ManualExposure + - AeLocked: + type: bool + description: | + Report the lock status of a running AE algorithm. + + If the AE algorithm is locked the value shall be set to true, if it's + converging it shall be set to false. If the AE algorithm is not + running the control shall not be present in the metadata control list. + + \sa AeEnable + - AwbEnable: type: bool description: | From patchwork Tue Oct 8 00:45:29 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2143 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id BBFC361968 for ; Tue, 8 Oct 2019 02:46:44 +0200 (CEST) X-Halon-ID: f5565c09-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f5565c09-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:43 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:29 +0200 Message-Id: <20191008004534.1585142-6-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 05/10] libcamera: request: Add metadata information X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:45 -0000 A new ControlList container is needed to hold metadata coming out of the IPA. The list of supported controls in this list is expected to grow, so for now do not add a validator for the list. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- include/libcamera/request.h | 2 ++ src/libcamera/request.cpp | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/include/libcamera/request.h b/include/libcamera/request.h index e3db5243aaf3cf30..2d5a5964e99eb75f 100644 --- a/include/libcamera/request.h +++ b/include/libcamera/request.h @@ -37,6 +37,7 @@ public: ~Request(); ControlList &controls() { return *controls_; } + ControlList &metadata() { return *metadata_; } const std::map &buffers() const { return bufferMap_; } int addBuffer(std::unique_ptr buffer); Buffer *findBuffer(Stream *stream) const; @@ -58,6 +59,7 @@ private: Camera *camera_; CameraControlValidator *validator_; ControlList *controls_; + ControlList *metadata_; std::map bufferMap_; std::unordered_set pending_; diff --git a/src/libcamera/request.cpp b/src/libcamera/request.cpp index 19f6d0b9a0aeb03c..23d3ab6f422c406e 100644 --- a/src/libcamera/request.cpp +++ b/src/libcamera/request.cpp @@ -65,6 +65,11 @@ Request::Request(Camera *camera, uint64_t cookie) */ validator_ = new CameraControlValidator(camera); controls_ = new ControlList(validator_); + + /** + * \todo: Add a validator for metadata controls. + */ + metadata_ = new ControlList(nullptr); } Request::~Request() @@ -74,6 +79,7 @@ Request::~Request() delete buffer; } + delete metadata_; delete controls_; delete validator_; } @@ -161,6 +167,14 @@ Buffer *Request::findBuffer(Stream *stream) const return it->second; } +/** + * \fn Request::metadata() + * \brief Retrieve the request's metadata + * \todo Offer a read-only API towards applications while keeping a read/write + * API internally. + * \return The metadata associated with the request + */ + /** * \fn Request::cookie() * \brief Retrieve the cookie set when the request was created From patchwork Tue Oct 8 00:45:30 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2144 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4DFD861966 for ; Tue, 8 Oct 2019 02:46:45 +0200 (CEST) X-Halon-ID: f5c8f0ba-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f5c8f0ba-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:44 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:30 +0200 Message-Id: <20191008004534.1585142-7-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 06/10] libcamera: ipa: meson: Allow access to internal libcamera headers X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:45 -0000 Allow IPA implementations to use internal libcamera headers so they can utilize helpers not exposed to applications. Signed-off-by: Niklas Söderlund Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/ipa/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipa/meson.build b/src/ipa/meson.build index f09915bc13887bd3..10448c2ffc76af4b 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -8,7 +8,7 @@ ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources ipa = shared_module(t[0], 'ipa_dummy.cpp', name_prefix : '', - include_directories : libcamera_includes, + include_directories : includes, install : true, install_dir : ipa_install_dir, cpp_args : '-DLICENSE="' + t[1] + '"') From patchwork Tue Oct 8 00:45:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2145 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 3D1CA6196C for ; Tue, 8 Oct 2019 02:46:47 +0200 (CEST) X-Halon-ID: f6430d9e-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f6430d9e-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:45 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:31 +0200 Message-Id: <20191008004534.1585142-8-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 07/10] libcamera: ipa: Extend to support IPA interactions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:47 -0000 The IPA interface needs to support interactions with the pipeline; add interfaces to control the sensor and handling of request ISP parameters and statistics. Signed-off-by: Niklas Söderlund --- include/ipa/ipa_interface.h | 38 +++++ src/ipa/ipa_dummy.cpp | 7 +- src/libcamera/ipa_interface.cpp | 195 ++++++++++++++++++++++++ src/libcamera/proxy/ipa_proxy_linux.cpp | 14 +- 4 files changed, 245 insertions(+), 9 deletions(-) diff --git a/include/ipa/ipa_interface.h b/include/ipa/ipa_interface.h index 2c5eb1fd524311cb..499340dc3a8c67c2 100644 --- a/include/ipa/ipa_interface.h +++ b/include/ipa/ipa_interface.h @@ -7,14 +7,52 @@ #ifndef __LIBCAMERA_IPA_INTERFACE_H__ #define __LIBCAMERA_IPA_INTERFACE_H__ +#include +#include + +#include +#include +#include +#include + +#include "v4l2_controls.h" + namespace libcamera { +struct IPAStream { + unsigned int pixelFormat; + Size size; +}; + +struct IPABuffer { + unsigned int type; + unsigned int id; + BufferMemory buffer; +}; + +struct IPAOperationData { + unsigned int operation; + std::vector data; + std::vector controls; + std::vector v4l2controls; + /* \todo: Add more vectors for data types used in pipa<->IPA interactions. */ +}; + class IPAInterface { public: virtual ~IPAInterface() {} virtual int init() = 0; + + virtual void configure(const std::map &streamConfig, + const std::map &entityControls) = 0; + + virtual void mapBuffers(const std::vector &buffers) = 0; + virtual void unmapBuffers(const std::vector &buffers) = 0; + + virtual void processEvent(const IPAOperationData &event) = 0; + Signal queueFrameAction; }; } /* namespace libcamera */ diff --git a/src/ipa/ipa_dummy.cpp b/src/ipa/ipa_dummy.cpp index 9d0cbdc8b1ad5565..08f74c4952f3923a 100644 --- a/src/ipa/ipa_dummy.cpp +++ b/src/ipa/ipa_dummy.cpp @@ -15,7 +15,12 @@ namespace libcamera { class IPADummy : public IPAInterface { public: - int init(); + int init() override; + void configure(const std::map &streamConfig, + const std::map &entityControls) override {} + void mapBuffers(const std::vector &buffers) override {} + void unmapBuffers(const std::vector &buffers) override {} + void processEvent(const IPAOperationData &event) override {} }; int IPADummy::init() diff --git a/src/libcamera/ipa_interface.cpp b/src/libcamera/ipa_interface.cpp index d7d8ca8881efcf66..c365d14c53669c21 100644 --- a/src/libcamera/ipa_interface.cpp +++ b/src/libcamera/ipa_interface.cpp @@ -14,9 +14,136 @@ namespace libcamera { +/** + * \struct IPAStream + * \brief Stream configuration for the IPA protocol + * + * The StreamConfiguration class contains too much information to be suitable + * for IPA protocols. The IPAStream structure mirrors the parameters from + * StreamConfiguration which are useful for the IPA. + */ + +/** + * \var IPAStream::pixelFormat + * \brief The pixel format + */ + +/** + * \var IPAStream::size + * \brief The size + */ + +/** + * \struct IPABuffer + * \brief Buffer information for the IPA protocol + * + * Buffers allocated by a pipeline handler might be needed inside the IPA. This + * structure allows the pipeline handler to associate a buffer with numerical + * ids that uniquely identify the buffer that can be used in the IPA protocol. + * + * The numerical identifiers are \a type and \a id. Type describes which buffer + * pool \a buffer belongs to and id unique identifies the buffer within that + * pool. + * + * \sa IPAInterface::mapBuffers() + * \sa IPAInterface::unmapBuffers() + */ + +/** + * \var IPABuffer::type + * \brief Type of the buffer + */ + +/** + * \var IPABuffer::id + * \brief ID of the buffer + */ + +/** + * \var IPABuffer::buffer + * \brief Hardware description of the buffer + * + * The description allows the IPA to map and unmap the buffer memory to access + * its content. + */ + +/** + * \struct IPAOperationData + * \brief Parameters for IPA operations + * + * A pipeline defines its IPA protocol by describing how an IPAOperationData + * structure shall be filled out for each of its operations. When the structure + * is populated it can be sent to the other part of the protocol and decoded. + * + * The \a operation describes which operation the receiver shall perform + * and implicitly describes which of the other fields in the structure are + * populated and how they shall be interpreted. All this is the responsibility + * of the pipeline implementation. + * + * \sa IPAInterface::processEvent() + * \sa IPAInterface::queueFrameAction + */ + +/** + * \var IPAOperationData::operation + * \brief Operation in the pipeline-to-IPA protocol + * + * The allocation of numerical values to operations are left to the + * implementation of the pipeline protocol. + */ + +/** + * \var IPAOperationData::data + * \brief Array of unsigned int values + * + * The interpretation and position of different values in the array are left + * to the implementation of the pipeline protocol. + */ + +/** + * \var IPAOperationData::controls + * \brief Array of ControlList + * + * The interpretation and position of different values in the array are left + * to the implementation of the pipeline protocol. + */ + +/** + * \var IPAOperationData::v4l2controls + * \brief Array of V4L2ControlList + * + * The interpretation and position of different values in the array are left + * to the implementation of the pipeline protocol. + */ + /** * \class IPAInterface * \brief Interface for IPA implementation + * + * Every pipeline implementation in libcamera may attach all or some of its + * cameras to an Image Processing Algorithm (IPA) plugin. An IPA plugin is + * developed for a specific pipeline and each pipeline might have multiple + * competing IPA implementations, both open and closed source. + * + * To allow for multiple and competing IPA plugins for the same pipeline, an + * interface for the pipeline and IPA communication is needed; IPAInterface + * is this interface. The interface defines how and what type of data can be + * exchanged, not how the data exchanged makes up a protocol. Each pipeline's + * design is unique and so is its IPA protocol. It is the pipeline's + * responsibility to define and document a protocol on top of the IPAInterface + * which can be used by multiple IPA plugins. + * + * The IPAInterface provides some methods to help the protocol design. As the + * requirements of the IPA plugins become more clear it is expected more + * common protocol operations will be identified and added. + * \todo Add more common operations if possible. + * + * The pipeline shall use the IPAManager to locate an IPAInterface compatible + * with the pipeline. The interface may then be used in the pipeline to interact + * with the IPA plugin to make up the protocol. + * \todo Add reference to how pipelines shall document their protocol. + * + * \sa IPAManager */ /** @@ -24,4 +151,72 @@ namespace libcamera { * \brief Initialise the IPAInterface */ +/** + * \fn IPAInterface::configure() + * \brief Configure the IPA stream and sensor settings + * \param[in] streamConfig List of stream configuration descriptions + * \param[in] entityControls List of controls provided by the pipeline entities + * + * This operation shall be called when the camera is started to inform the IPA + * of the camera's streams and the sensor settings. + */ + +/** + * \fn IPAInterface::mapBuffers() + * \brief Map the buffers shared by the pipeline to the IPA + * \param[in] buffers List of buffers to map + * + * This operation is used to inform the IPA module of the memory buffers set up + * by the pipeline handler and associate to each of them a numerical \a type and + * \a id. All buffers that the pipeline wishes to share with an IPA must be + * mapped in the IPA with this operation. + * + * After the buffers have been initialized a specific buffer can be referenced + * using the numerical \a type and \a id provided when the buffers were mapped. + * + * The numerical values for type and id used to describe a buffer have no + * meaning in the core libcamera code and the interface is pipeline handler + * specific. + * + * \sa unmapBuffers() + * \todo Can we make this a generic function? + */ + +/** + * \fn IPAInterface::unmapBuffers() + * \brief Unmap the buffers shared by the pipeline to the IPA + * \param[in] buffers List of buffers to unmap + * + * This operation removes the mapping set up with mapBuffers(). + * + * \sa mapBuffers() + * \todo Can we make this a generic function? + */ + +/** + * \fn IPAInterface::processEvent() + * \brief Process an event from the pipeline handler + * \param[in] event Event to process + * + * This operation is used by pipeline handlers to inform the IPA module of + * events that occurred during the on-going capture operation. + * + * Each \a event notified by the pipeline handler with this method is handled + * by the IPA which interprets the operation parameters in an implementation + * specific way, which needs to be separately documented. + */ + +/** + * \fn IPAInterface::queueFrameAction() + * \brief Queue an action associated with a frame to the pipeline handler + * \param[in] frame The frame number for the action + * \param[in] controls List of controls associated with the action + * + * This signal is emitted when the IPA wishes to queue a FrameAction on the + * pipeline. The pipeline is still responsible for the scheduling of the action + * on its timeline. + * + * The IPA operation describing the frame action is passed as a parameter. + */ + } /* namespace libcamera */ diff --git a/src/libcamera/proxy/ipa_proxy_linux.cpp b/src/libcamera/proxy/ipa_proxy_linux.cpp index 62fcb529e1c7e4ff..14e8bb6067623fc6 100644 --- a/src/libcamera/proxy/ipa_proxy_linux.cpp +++ b/src/libcamera/proxy/ipa_proxy_linux.cpp @@ -26,7 +26,12 @@ public: IPAProxyLinux(IPAModule *ipam); ~IPAProxyLinux(); - int init(); + int init() override { return 0; } + void configure(const std::map &streamConfig, + const std::map &entityControls) override {} + void mapBuffers(const std::vector &buffers) override {} + void unmapBuffers(const std::vector &buffers) override {} + void processEvent(const IPAOperationData &event) override {} private: void readyRead(IPCUnixSocket *ipc); @@ -36,13 +41,6 @@ private: IPCUnixSocket *socket_; }; -int IPAProxyLinux::init() -{ - LOG(IPAProxy, Debug) << "initializing IPA via dummy proxy!"; - - return 0; -} - IPAProxyLinux::IPAProxyLinux(IPAModule *ipam) : proc_(nullptr), socket_(nullptr) { From patchwork Tue Oct 8 00:45:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2146 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D9E6C6196E for ; Tue, 8 Oct 2019 02:46:48 +0200 (CEST) X-Halon-ID: f6e32d6e-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f6e32d6e-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:46 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:32 +0200 Message-Id: <20191008004534.1585142-9-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 08/10] include: linux: Add rkisp1 kernel header and format definitions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:49 -0000 Add kernel header and format definitions from v8 of the rkisp1 series [1]. The driver is currently out of tree, the header file is not exported as part of the standard kernel uAPI. Headers have been exported on top of a v5.2 kernel tree using scripts/headers_install.sh. 1. https://lore.kernel.org/linux-media/20190730184256.30338-1-helen.koike@collabora.com/ Signed-off-by: Niklas Söderlund Acked-by: Kieran Bingham Acked-by: Laurent Pinchart --- include/linux/rkisp1-config.h | 816 ++++++++++++++++++++++++++++++++++ include/linux/videodev2.h | 4 + 2 files changed, 820 insertions(+) create mode 100644 include/linux/rkisp1-config.h diff --git a/include/linux/rkisp1-config.h b/include/linux/rkisp1-config.h new file mode 100644 index 0000000000000000..f0d1bd82e857fd79 --- /dev/null +++ b/include/linux/rkisp1-config.h @@ -0,0 +1,816 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip isp1 driver + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +/* + * TODO: Improve documentation, mostly regarding abbreviation and hardware + * specificities. + */ + +#ifndef _RKISP1_CONFIG_H +#define _RKISP1_CONFIG_H + +#include +#include + +#define CIFISP_MODULE_DPCC (1 << 0) +#define CIFISP_MODULE_BLS (1 << 1) +#define CIFISP_MODULE_SDG (1 << 2) +#define CIFISP_MODULE_HST (1 << 3) +#define CIFISP_MODULE_LSC (1 << 4) +#define CIFISP_MODULE_AWB_GAIN (1 << 5) +#define CIFISP_MODULE_FLT (1 << 6) +#define CIFISP_MODULE_BDM (1 << 7) +#define CIFISP_MODULE_CTK (1 << 8) +#define CIFISP_MODULE_GOC (1 << 9) +#define CIFISP_MODULE_CPROC (1 << 10) +#define CIFISP_MODULE_AFC (1 << 11) +#define CIFISP_MODULE_AWB (1 << 12) +#define CIFISP_MODULE_IE (1 << 13) +#define CIFISP_MODULE_AEC (1 << 14) +#define CIFISP_MODULE_WDR (1 << 15) +#define CIFISP_MODULE_DPF (1 << 16) +#define CIFISP_MODULE_DPF_STRENGTH (1 << 17) + +#define CIFISP_CTK_COEFF_MAX 0x100 +#define CIFISP_CTK_OFFSET_MAX 0x800 + +#define CIFISP_AE_MEAN_MAX 25 +#define CIFISP_HIST_BIN_N_MAX 16 +#define CIFISP_AFM_MAX_WINDOWS 3 +#define CIFISP_DEGAMMA_CURVE_SIZE 17 + +#define CIFISP_BDM_MAX_TH 0xFF + +/* + * Black level compensation + */ +/* maximum value for horizontal start address */ +#define CIFISP_BLS_START_H_MAX 0x00000FFF +/* maximum value for horizontal stop address */ +#define CIFISP_BLS_STOP_H_MAX 0x00000FFF +/* maximum value for vertical start address */ +#define CIFISP_BLS_START_V_MAX 0x00000FFF +/* maximum value for vertical stop address */ +#define CIFISP_BLS_STOP_V_MAX 0x00000FFF +/* maximum is 2^18 = 262144*/ +#define CIFISP_BLS_SAMPLES_MAX 0x00000012 +/* maximum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MAX 0x00000FFF +/* minimum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MIN 0xFFFFF000 +/* 13 bit range (signed)*/ +#define CIFISP_BLS_FIX_MASK 0x00001FFF + +/* + * Automatic white balance measurments + */ +#define CIFISP_AWB_MAX_GRID 1 +#define CIFISP_AWB_MAX_FRAMES 7 + +/* + * Gamma out + */ +/* Maximum number of color samples supported */ +#define CIFISP_GAMMA_OUT_MAX_SAMPLES 17 + +/* + * Lens shade correction + */ +#define CIFISP_LSC_GRAD_TBL_SIZE 8 +#define CIFISP_LSC_SIZE_TBL_SIZE 8 +/* + * The following matches the tuning process, + * not the max capabilities of the chip. + * Last value unused. + */ +#define CIFISP_LSC_DATA_TBL_SIZE 290 + +/* + * Histogram calculation + */ +/* Last 3 values unused. */ +#define CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE 28 + +/* + * Defect Pixel Cluster Correction + */ +#define CIFISP_DPCC_METHODS_MAX 3 + +/* + * Denoising pre filter + */ +#define CIFISP_DPF_MAX_NLF_COEFFS 17 +#define CIFISP_DPF_MAX_SPATIAL_COEFFS 6 + +/* + * Measurement types + */ +#define CIFISP_STAT_AWB (1 << 0) +#define CIFISP_STAT_AUTOEXP (1 << 1) +#define CIFISP_STAT_AFM_FIN (1 << 2) +#define CIFISP_STAT_HIST (1 << 3) + +enum cifisp_histogram_mode { + CIFISP_HISTOGRAM_MODE_DISABLE, + CIFISP_HISTOGRAM_MODE_RGB_COMBINED, + CIFISP_HISTOGRAM_MODE_R_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_G_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_B_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_Y_HISTOGRAM +}; + +enum cifisp_awb_mode_type { + CIFISP_AWB_MODE_MANUAL, + CIFISP_AWB_MODE_RGB, + CIFISP_AWB_MODE_YCBCR +}; + +enum cifisp_flt_mode { + CIFISP_FLT_STATIC_MODE, + CIFISP_FLT_DYNAMIC_MODE +}; + +/** + * enum cifisp_exp_ctrl_autostop - stop modes + * @CIFISP_EXP_CTRL_AUTOSTOP_0: continuous measurement + * @CIFISP_EXP_CTRL_AUTOSTOP_1: stop measuring after a complete frame + */ +enum cifisp_exp_ctrl_autostop { + CIFISP_EXP_CTRL_AUTOSTOP_0 = 0, + CIFISP_EXP_CTRL_AUTOSTOP_1 = 1, +}; + +/** + * enum cifisp_exp_meas_mode - Exposure measure mode + * @CIFISP_EXP_MEASURING_MODE_0: Y = 16 + 0.25R + 0.5G + 0.1094B + * @CIFISP_EXP_MEASURING_MODE_1: Y = (R + G + B) x (85/256) + */ +enum cifisp_exp_meas_mode { + CIFISP_EXP_MEASURING_MODE_0, + CIFISP_EXP_MEASURING_MODE_1, +}; + +/*---------- PART1: Input Parameters ------------*/ + +struct cifisp_window { + __u16 h_offs; + __u16 v_offs; + __u16 h_size; + __u16 v_size; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_fixed_val - BLS fixed subtraction values + * + * The values will be subtracted from the sensor + * values. Therefore a negative value means addition instead of subtraction! + * + * @r: Fixed (signed!) subtraction value for Bayer pattern R + * @gr: Fixed (signed!) subtraction value for Bayer pattern Gr + * @gb: Fixed (signed!) subtraction value for Bayer pattern Gb + * @b: Fixed (signed!) subtraction value for Bayer pattern B + */ +struct cifisp_bls_fixed_val { + __s16 r; + __s16 gr; + __s16 gb; + __s16 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_config - Configuration used by black level subtraction + * + * @enable_auto: Automatic mode activated means that the measured values + * are subtracted. Otherwise the fixed subtraction + * values will be subtracted. + * @en_windows: enabled window + * @bls_window1: Measurement window 1 size + * @bls_window2: Measurement window 2 size + * @bls_samples: Set amount of measured pixels for each Bayer position + * (A, B,C and D) to 2^bls_samples. + * @cifisp_bls_fixed_val: Fixed subtraction values + */ +struct cifisp_bls_config { + __u8 enable_auto; + __u8 en_windows; + struct cifisp_window bls_window1; + struct cifisp_window bls_window2; + __u8 bls_samples; + struct cifisp_bls_fixed_val fixed_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Methods Configuration used by DPCC + * + * Methods Configuration used by Defect Pixel Cluster Correction + * + * @method: Method enable bits + * @line_thresh: Line threshold + * @line_mad_fac: Line MAD factor + * @pg_fac: Peak gradient factor + * @rnd_thresh: Rank Neighbor Difference threshold + * @rg_fac: Rank gradient factor + */ +struct cifisp_dpcc_methods_config { + __u32 method; + __u32 line_thresh; + __u32 line_mad_fac; + __u32 pg_fac; + __u32 rnd_thresh; + __u32 rg_fac; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Configuration used by DPCC + * + * Configuration used by Defect Pixel Cluster Correction + * + * @mode: dpcc output mode + * @output_mode: whether use hard coded methods + * @set_use: stage1 methods set + * @methods: methods config + * @ro_limits: rank order limits + * @rnd_offs: differential rank offsets for rank neighbor difference + */ +struct cifisp_dpcc_config { + __u32 mode; + __u32 output_mode; + __u32 set_use; + struct cifisp_dpcc_methods_config methods[CIFISP_DPCC_METHODS_MAX]; + __u32 ro_limits; + __u32 rnd_offs; +} __attribute__ ((packed)); + +struct cifisp_gamma_corr_curve { + __u16 gamma_y[CIFISP_DEGAMMA_CURVE_SIZE]; +} __attribute__ ((packed)); + +struct cifisp_gamma_curve_x_axis_pnts { + __u32 gamma_dx0; + __u32 gamma_dx1; +} __attribute__ ((packed)); + +/** + * struct cifisp_gamma_corr_curve - Configuration used by sensor degamma + * + * @curve_x: gamma curve point definition axis for x + * @xa_pnts: x increments + */ +struct cifisp_sdg_config { + struct cifisp_gamma_corr_curve curve_r; + struct cifisp_gamma_corr_curve curve_g; + struct cifisp_gamma_corr_curve curve_b; + struct cifisp_gamma_curve_x_axis_pnts xa_pnts; +} __attribute__ ((packed)); + +/** + * struct cifisp_lsc_config - Configuration used by Lens shading correction + * + * refer to REF_01 for details + */ +struct cifisp_lsc_config { + __u32 r_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gr_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gb_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 b_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + + __u32 x_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + __u32 y_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + + __u32 x_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u32 y_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u16 config_width; + __u16 config_height; +} __attribute__ ((packed)); + +/** + * struct cifisp_ie_config - Configuration used by image effects + * + * @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1 + * @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2 + * @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1 + * @eff_mat_4: 3x3 Matrix Coefficients for Sketch Effect 2 + * @eff_mat_5: 3x3 Matrix Coefficients for Sketch Effect 3 + * @eff_tint: Chrominance increment values of tint (used for sepia effect) + */ +struct cifisp_ie_config { + __u16 effect; + __u16 color_sel; + __u16 eff_mat_1; + __u16 eff_mat_2; + __u16 eff_mat_3; + __u16 eff_mat_4; + __u16 eff_mat_5; + __u16 eff_tint; +} __attribute__ ((packed)); + +/** + * struct cifisp_cproc_config - Configuration used by Color Processing + * + * @c_out_range: Chrominance pixel clipping range at output. + * (0 for limit, 1 for full) + * @y_in_range: Luminance pixel clipping range at output. + * @y_out_range: Luminance pixel clipping range at output. + * @contrast: 00~ff, 0.0~1.992 + * @brightness: 80~7F, -128~+127 + * @sat: saturation, 00~FF, 0.0~1.992 + * @hue: 80~7F, -90~+87.188 + */ +struct cifisp_cproc_config { + __u8 c_out_range; + __u8 y_in_range; + __u8 y_out_range; + __u8 contrast; + __u8 brightness; + __u8 sat; + __u8 hue; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_meas_config - Configuration used by auto white balance + * + * @awb_wnd: white balance measurement window (in pixels) + * (from enum cifisp_awb_mode_type) + * @max_y: only pixels values < max_y contribute to awb measurement, set to 0 + * to disable this feature + * @min_y: only pixels values > min_y contribute to awb measurement + * @max_csum: Chrominance sum maximum value, only consider pixels with Cb+Cr, + * smaller than threshold for awb measurements + * @min_c: Chrominance minimum value, only consider pixels with Cb/Cr + * each greater than threshold value for awb measurements + * @frames: number of frames - 1 used for mean value calculation + * (ucFrames=0 means 1 Frame) + * @awb_ref_cr: reference Cr value for AWB regulation, target for AWB + * @awb_ref_cb: reference Cb value for AWB regulation, target for AWB + */ +struct cifisp_awb_meas_config { + /* + * Note: currently the h and v offsets are mapped to grid offsets + */ + struct cifisp_window awb_wnd; + __u32 awb_mode; + __u8 max_y; + __u8 min_y; + __u8 max_csum; + __u8 min_c; + __u8 frames; + __u8 awb_ref_cr; + __u8 awb_ref_cb; + __u8 enable_ymax_cmp; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_gain_config - Configuration used by auto white balance gain + * + * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8 + */ +struct cifisp_awb_gain_config { + __u16 gain_red; + __u16 gain_green_r; + __u16 gain_blue; + __u16 gain_green_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_flt_config - Configuration used by ISP filtering + * + * @mode: ISP_FILT_MODE register fields (from enum cifisp_flt_mode) + * @grn_stage1: ISP_FILT_MODE register fields + * @chr_h_mode: ISP_FILT_MODE register fields + * @chr_v_mode: ISP_FILT_MODE register fields + * + * refer to REF_01 for details. + */ + +struct cifisp_flt_config { + __u32 mode; + __u8 grn_stage1; + __u8 chr_h_mode; + __u8 chr_v_mode; + __u32 thresh_bl0; + __u32 thresh_bl1; + __u32 thresh_sh0; + __u32 thresh_sh1; + __u32 lum_weight; + __u32 fac_sh1; + __u32 fac_sh0; + __u32 fac_mid; + __u32 fac_bl0; + __u32 fac_bl1; +} __attribute__ ((packed)); + +/** + * struct cifisp_bdm_config - Configuration used by Bayer DeMosaic + * + * @demosaic_th: threshod for bayer demosaicing texture detection + */ +struct cifisp_bdm_config { + __u8 demosaic_th; +} __attribute__ ((packed)); + +/** + * struct cifisp_ctk_config - Configuration used by Cross Talk correction + * + * @coeff: color correction matrix + * @ct_offset_b: offset for the crosstalk correction matrix + */ +struct cifisp_ctk_config { + __u16 coeff0; + __u16 coeff1; + __u16 coeff2; + __u16 coeff3; + __u16 coeff4; + __u16 coeff5; + __u16 coeff6; + __u16 coeff7; + __u16 coeff8; + __u16 ct_offset_r; + __u16 ct_offset_g; + __u16 ct_offset_b; +} __attribute__ ((packed)); + +enum cifisp_goc_mode { + CIFISP_GOC_MODE_LOGARITHMIC, + CIFISP_GOC_MODE_EQUIDISTANT +}; + +/** + * struct cifisp_goc_config - Configuration used by Gamma Out correction + * + * @mode: goc mode (from enum cifisp_goc_mode) + * @gamma_y: gamma out curve y-axis for all color components + */ +struct cifisp_goc_config { + __u32 mode; + __u16 gamma_y[CIFISP_GAMMA_OUT_MAX_SAMPLES]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hst_config - Configuration used by Histogram + * + * @mode: histogram mode (from enum cifisp_histogram_mode) + * @histogram_predivider: process every stepsize pixel, all other pixels are + * skipped + * @meas_window: coordinates of the measure window + * @hist_weight: weighting factor for sub-windows + */ +struct cifisp_hst_config { + __u32 mode; + __u8 histogram_predivider; + struct cifisp_window meas_window; + __u8 hist_weight[CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE]; +} __attribute__ ((packed)); + +/** + * struct cifisp_aec_config - Configuration used by Auto Exposure Control + * + * @mode: Exposure measure mode (from enum cifisp_exp_meas_mode) + * @autostop: stop mode (from enum cifisp_exp_ctrl_autostop) + * @meas_window: coordinates of the measure window + */ +struct cifisp_aec_config { + __u32 mode; + __u32 autostop; + struct cifisp_window meas_window; +} __attribute__ ((packed)); + +/** + * struct cifisp_afc_config - Configuration used by Auto Focus Control + * + * @num_afm_win: max CIFISP_AFM_MAX_WINDOWS + * @afm_win: coordinates of the meas window + * @thres: threshold used for minimizing the influence of noise + * @var_shift: the number of bits for the shift operation at the end of the + * calculation chain. + */ +struct cifisp_afc_config { + __u8 num_afm_win; + struct cifisp_window afm_win[CIFISP_AFM_MAX_WINDOWS]; + __u32 thres; + __u32 var_shift; +} __attribute__ ((packed)); + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_GAIN_USAGE_DISABLED: don't use any gains in preprocessing stage + * @CIFISP_DPF_GAIN_USAGE_NF_GAINS: use only the noise function gains from + * registers DPF_NF_GAIN_R, ... + * @CIFISP_DPF_GAIN_USAGE_LSC_GAINS: use only the gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS: use the noise function gains and the + * gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_AWB_GAINS: use only the gains from AWB module + * @CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: use the gains from AWB and LSC module + * @CIFISP_DPF_GAIN_USAGE_MAX: upper border (only for an internal evaluation) + */ +enum cifisp_dpf_gain_usage { + CIFISP_DPF_GAIN_USAGE_DISABLED, + CIFISP_DPF_GAIN_USAGE_NF_GAINS, + CIFISP_DPF_GAIN_USAGE_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_MAX +}; + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_RB_FILTERSIZE_13x9: red and blue filter kernel size 13x9 + * (means 7x5 active pixel) + * @CIFISP_DPF_RB_FILTERSIZE_9x9: red and blue filter kernel size 9x9 + * (means 5x5 active pixel) + */ +enum cifisp_dpf_rb_filtersize { + CIFISP_DPF_RB_FILTERSIZE_13x9, + CIFISP_DPF_RB_FILTERSIZE_9x9, +}; + +/** + * enum cifisp_dpf_nll_scale_mode - dpf noise level scale mode + * @CIFISP_NLL_SCALE_LINEAR: use a linear scaling + * @CIFISP_NLL_SCALE_LOGARITHMIC: use a logarithmic scaling + */ +enum cifisp_dpf_nll_scale_mode { + CIFISP_NLL_SCALE_LINEAR, + CIFISP_NLL_SCALE_LOGARITHMIC, +}; + +/** + * struct cifisp_dpf_nll - Noise level lookup + * + * @coeff: Noise level Lookup coefficient + * @scale_mode: dpf noise level scale mode (from enum cifisp_dpf_nll_scale_mode) + */ +struct cifisp_dpf_nll { + __u16 coeff[CIFISP_DPF_MAX_NLF_COEFFS]; + __u32 scale_mode; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_rb_flt - Red blue filter config + * + * @fltsize: The filter size for the red and blue pixels + * (from enum cifisp_dpf_rb_filtersize) + * @spatial_coeff: Spatial weights + * @r_enable: enable filter processing for red pixels + * @b_enable: enable filter processing for blue pixels + */ +struct cifisp_dpf_rb_flt { + __u32 fltsize; + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 r_enable; + __u8 b_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_g_flt - Green filter Configuration + * + * @spatial_coeff: Spatial weights + * @gr_enable: enable filter processing for green pixels in green/red lines + * @gb_enable: enable filter processing for green pixels in green/blue lines + */ +struct cifisp_dpf_g_flt { + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 gr_enable; + __u8 gb_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_gain - Noise function Configuration + * + * @mode: dpf gain usage (from enum cifisp_dpf_gain_usage) + * @nf_r_gain: Noise function Gain that replaces the AWB gain for red pixels + * @nf_b_gain: Noise function Gain that replaces the AWB gain for blue pixels + * @nf_gr_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a red line + * @nf_gb_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a blue line + */ +struct cifisp_dpf_gain { + __u32 mode; + __u16 nf_r_gain; + __u16 nf_b_gain; + __u16 nf_gr_gain; + __u16 nf_gb_gain; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_config - Configuration used by De-noising pre-filter + * + * @gain: noise function gain + * @g_flt: green filter config + * @rb_flt: red blue filter config + * @nll: noise level lookup + */ +struct cifisp_dpf_config { + struct cifisp_dpf_gain gain; + struct cifisp_dpf_g_flt g_flt; + struct cifisp_dpf_rb_flt rb_flt; + struct cifisp_dpf_nll nll; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_strength_config - strength of the filter + * + * @r: filter strength of the RED filter + * @g: filter strength of the GREEN filter + * @b: filter strength of the BLUE filter + */ +struct cifisp_dpf_strength_config { + __u8 r; + __u8 g; + __u8 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_other_cfg - Parameters for some blocks in rockchip isp1 + * + * @dpcc_config: Defect Pixel Cluster Correction config + * @bls_config: Black Level Subtraction config + * @sdg_config: sensor degamma config + * @lsc_config: Lens Shade config + * @awb_gain_config: Auto White balance gain config + * @flt_config: filter config + * @bdm_config: demosaic config + * @ctk_config: cross talk config + * @goc_config: gamma out config + * @bls_config: black level subtraction config + * @dpf_config: De-noising pre-filter config + * @dpf_strength_config: dpf strength config + * @cproc_config: color process config + * @ie_config: image effects config + */ +struct cifisp_isp_other_cfg { + struct cifisp_dpcc_config dpcc_config; + struct cifisp_bls_config bls_config; + struct cifisp_sdg_config sdg_config; + struct cifisp_lsc_config lsc_config; + struct cifisp_awb_gain_config awb_gain_config; + struct cifisp_flt_config flt_config; + struct cifisp_bdm_config bdm_config; + struct cifisp_ctk_config ctk_config; + struct cifisp_goc_config goc_config; + struct cifisp_dpf_config dpf_config; + struct cifisp_dpf_strength_config dpf_strength_config; + struct cifisp_cproc_config cproc_config; + struct cifisp_ie_config ie_config; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_meas_cfg - Rockchip ISP1 Measure Parameters + * + * @awb_meas_config: auto white balance config + * @hst_config: histogram config + * @aec_config: auto exposure config + * @afc_config: auto focus config + */ +struct cifisp_isp_meas_cfg { + struct cifisp_awb_meas_config awb_meas_config; + struct cifisp_hst_config hst_config; + struct cifisp_aec_config aec_config; + struct cifisp_afc_config afc_config; +} __attribute__ ((packed)); + +/** + * struct rkisp1_isp_params_cfg - Rockchip ISP1 Input Parameters Meta Data + * + * @module_en_update: mask the enable bits of which module should be updated + * @module_ens: mask the enable value of each module, only update the module + * which correspond bit was set in module_en_update + * @module_cfg_update: mask the config bits of which module should be updated + * @meas: measurement config + * @others: other config + */ +struct rkisp1_isp_params_cfg { + __u32 module_en_update; + __u32 module_ens; + __u32 module_cfg_update; + + struct cifisp_isp_meas_cfg meas; + struct cifisp_isp_other_cfg others; +} __attribute__ ((packed)); + +/*---------- PART2: Measurement Statistics ------------*/ + +/** + * struct cifisp_bls_meas_val - AWB measured values + * + * @cnt: White pixel count, number of "white pixels" found during laster + * measurement + * @mean_y_or_g: Mean value of Y within window and frames, + * Green if RGB is selected. + * @mean_cb_or_b: Mean value of Cb within window and frames, + * Blue if RGB is selected. + * @mean_cr_or_r: Mean value of Cr within window and frames, + * Red if RGB is selected. + */ +struct cifisp_awb_meas { + __u32 cnt; + __u8 mean_y_or_g; + __u8 mean_cb_or_b; + __u8 mean_cr_or_r; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_stat - statistics automatic white balance data + * + * @awb_mean: Mean measured data + */ +struct cifisp_awb_stat { + struct cifisp_awb_meas awb_mean[CIFISP_AWB_MAX_GRID]; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_meas_val - BLS measured values + * + * @meas_r: Mean measured value for Bayer pattern R + * @meas_gr: Mean measured value for Bayer pattern Gr + * @meas_gb: Mean measured value for Bayer pattern Gb + * @meas_b: Mean measured value for Bayer pattern B + */ +struct cifisp_bls_meas_val { + __u16 meas_r; + __u16 meas_gr; + __u16 meas_gb; + __u16 meas_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_ae_stat - statistics auto exposure data + * + * @exp_mean: Mean luminance value of block xx + * @bls_val: BLS measured values + * + * Image is divided into 5x5 blocks. + */ +struct cifisp_ae_stat { + __u8 exp_mean[CIFISP_AE_MEAN_MAX]; + struct cifisp_bls_meas_val bls_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_meas_val - AF measured values + * + * @sum: sharpness, refer to REF_01 for definition + * @lum: luminance, refer to REF_01 for definition + */ +struct cifisp_af_meas_val { + __u32 sum; + __u32 lum; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_stat - statistics auto focus data + * + * @window: AF measured value of window x + * + * The module measures the sharpness in 3 windows of selectable size via + * register settings(ISP_AFM_*_A/B/C) + */ +struct cifisp_af_stat { + struct cifisp_af_meas_val window[CIFISP_AFM_MAX_WINDOWS]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hist_stat - statistics histogram data + * + * @hist_bins: measured bin counters + * + * Measurement window divided into 25 sub-windows, set + * with ISP_HIST_XXX + */ +struct cifisp_hist_stat { + __u16 hist_bins[CIFISP_HIST_BIN_N_MAX]; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Data + * + * @cifisp_awb_stat: statistics data for automatic white balance + * @cifisp_ae_stat: statistics data for auto exposure + * @cifisp_af_stat: statistics data for auto focus + * @cifisp_hist_stat: statistics histogram data + */ +struct cifisp_stat { + struct cifisp_awb_stat awb; + struct cifisp_ae_stat ae; + struct cifisp_af_stat af; + struct cifisp_hist_stat hist; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Meta Data + * + * @meas_type: measurement types (CIFISP_STAT_ definitions) + * @frame_id: frame ID for sync + * @params: statistics data + */ +struct rkisp1_stat_buffer { + __u32 meas_type; + __u32 frame_id; + struct cifisp_stat params; +} __attribute__ ((packed)); + +#endif /* _RKISP1_CONFIG_H */ diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index 5e739270116d3a21..94304bfeb4b6b9ed 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -727,6 +727,10 @@ struct v4l2_pix_format { #define V4L2_META_FMT_UVC v4l2_fourcc('U', 'V', 'C', 'H') /* UVC Payload Header metadata */ #define V4L2_META_FMT_D4XX v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */ +/* Vendor specific - used for RK_ISP1 camera sub-system */ +#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P') /* Rockchip ISP1 params */ +#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A statistics */ + /* priv field value to indicates that subsequent fields are valid. */ #define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe From patchwork Tue Oct 8 00:45:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2147 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4D83F6196E for ; Tue, 8 Oct 2019 02:46:49 +0200 (CEST) X-Halon-ID: f7fbcbe4-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f7fbcbe4-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:48 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:33 +0200 Message-Id: <20191008004534.1585142-10-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 09/10] libcamera: ipa: rkisp1: Add basic control of auto exposure X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:49 -0000 Add an IPA which controls the exposure time and analog gain for a sensor connected to the rkisp1 pipeline. The IPA supports turning AE on and off and informing the camera of the status of the AE control loop. Signed-off-by: Niklas Söderlund --- include/ipa/rkisp1.h | 23 ++++ src/ipa/ipa_rkisp1.cpp | 281 +++++++++++++++++++++++++++++++++++++++++ src/ipa/meson.build | 13 ++ 3 files changed, 317 insertions(+) create mode 100644 include/ipa/rkisp1.h create mode 100644 src/ipa/ipa_rkisp1.cpp diff --git a/include/ipa/rkisp1.h b/include/ipa/rkisp1.h new file mode 100644 index 0000000000000000..7db50afa6bef6e61 --- /dev/null +++ b/include/ipa/rkisp1.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Image Processing Algorithm interface for RkISP1 + */ +#ifndef __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ +#define __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ + +enum RkISP1BufferType { + RKISP1_BUFFER_PARAM = 1, + RKISP1_BUFFER_STAT = 2, +}; + +enum RkISP1Operations { + RKISP1_IPA_ACTION_V4L2_SET = 1, + RKISP1_IPA_ACTION_QUEUE_BUFFER = 2, + RKISP1_IPA_ACTION_METADATA = 3, + RKISP1_IPA_EVENT_SIGNAL_BUFFER = 4, + RKISP1_IPA_EVENT_QUEUE_REQUEST = 5, +}; + +#endif /* __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ */ diff --git a/src/ipa/ipa_rkisp1.cpp b/src/ipa/ipa_rkisp1.cpp new file mode 100644 index 0000000000000000..0cc0772422e1ea51 --- /dev/null +++ b/src/ipa/ipa_rkisp1.cpp @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * ipa_rkisp1.cpp - RkISP1 Image Processing Algorithms + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "utils.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPARkISP1) + +class IPARkISP1 : public IPAInterface +{ +public: + int init() override { return 0; } + + void configure(const std::map &streamConfig, + const std::map &entityControls) override; + void mapBuffers(const std::vector &buffers) override; + void unmapBuffers(const std::vector &buffers) override; + void processEvent(const IPAOperationData &event) override; + +private: + void signalBuffer(unsigned int type, unsigned int id); + void queueRequest(unsigned int frame, const ControlList &controls); + void updateStatistics(unsigned int frame, BufferMemory &statistics); + + void setControls(unsigned int frame); + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metadataReady(unsigned int frame, unsigned int aeState); + + std::map> bufferInfo_; + std::map> bufferFrame_; + std::map> bufferFree_; + + /* Camera sensor controls. */ + bool autoExposure_; + uint64_t exposure_; + uint64_t minExposure_; + uint64_t maxExposure_; + uint64_t gain_; + uint64_t minGain_; + uint64_t maxGain_; +}; + +void IPARkISP1::configure(const std::map &streamConfig, + const std::map &entityControls) +{ + if (entityControls.empty()) + return; + + const V4L2ControlInfoMap &ctrls = entityControls.at(0); + + const auto itExp = ctrls.find(V4L2_CID_EXPOSURE); + if (itExp == ctrls.end()) { + LOG(IPARkISP1, Error) << "Can't find exposure control"; + return; + } + + const auto itGain = ctrls.find(V4L2_CID_ANALOGUE_GAIN); + if (itGain == ctrls.end()) { + LOG(IPARkISP1, Error) << "Can't find gain control"; + return; + } + + autoExposure_ = true; + + minExposure_ = std::max(itExp->second.range().min().get(), 1); + maxExposure_ = itExp->second.range().max().get(); + exposure_ = minExposure_; + + minGain_ = std::max(itGain->second.range().min().get(), 1); + maxGain_ = itGain->second.range().max().get(); + gain_ = minGain_; + + LOG(IPARkISP1, Info) + << "Exposure: " << minExposure_ << "-" << maxExposure_ + << " Gain: " << minGain_ << "-" << maxGain_; + + setControls(0); +} + +void IPARkISP1::mapBuffers(const std::vector &buffers) +{ + for (IPABuffer buffer : buffers) { + bufferInfo_[buffer.type][buffer.id] = buffer.buffer; + bufferInfo_[buffer.type][buffer.id].planes()[0].mem(); + bufferFree_[buffer.type].push(buffer.id); + } +} + +void IPARkISP1::unmapBuffers(const std::vector &buffers) +{ + for (IPABuffer buffer : buffers) + bufferInfo_[buffer.type].erase(buffer.id); +} + +void IPARkISP1::processEvent(const IPAOperationData &event) +{ + switch (event.operation) { + case RKISP1_IPA_EVENT_SIGNAL_BUFFER: + signalBuffer(event.data[0], event.data[1]); + break; + case RKISP1_IPA_EVENT_QUEUE_REQUEST: + queueRequest(event.data[0], event.controls[0]); + break; + default: + LOG(IPARkISP1, Error) << "Unkown event " << event.operation; + break; + } +} + +void IPARkISP1::signalBuffer(unsigned int type, unsigned int id) +{ + if (type == RKISP1_BUFFER_STAT) { + unsigned int frame = bufferFrame_[type][id]; + BufferMemory &mem = bufferInfo_[type][id]; + updateStatistics(frame, mem); + } + + bufferFree_[type].push(id); +} + +void IPARkISP1::queueRequest(unsigned int frame, const ControlList &controls) +{ + /* Find buffers. */ + if (bufferFree_[RKISP1_BUFFER_PARAM].empty()) { + LOG(IPARkISP1, Error) << "Param buffer underrun"; + return; + } + + if (bufferFree_[RKISP1_BUFFER_STAT].empty()) { + LOG(IPARkISP1, Error) << "Statistics buffer underrun"; + return; + } + + unsigned int paramid = bufferFree_[RKISP1_BUFFER_PARAM].front(); + bufferFree_[RKISP1_BUFFER_PARAM].pop(); + unsigned int statid = bufferFree_[RKISP1_BUFFER_STAT].front(); + bufferFree_[RKISP1_BUFFER_STAT].pop(); + + bufferFrame_[RKISP1_BUFFER_PARAM][paramid] = frame; + bufferFrame_[RKISP1_BUFFER_STAT][statid] = frame; + + /* Prepare parameters buffer. */ + BufferMemory &mem = bufferInfo_[RKISP1_BUFFER_PARAM][paramid]; + rkisp1_isp_params_cfg *params = + static_cast(mem.planes()[0].mem()); + + memset(params, 0, sizeof(*params)); + + /* Auto Exposure on/off. */ + if (controls.contains(controls::AeEnable)) { + autoExposure_ = controls.get(controls::AeEnable); + if (autoExposure_) + params->module_ens = CIFISP_MODULE_AEC; + + params->module_en_update = CIFISP_MODULE_AEC; + } + + /* Queue buffers to pipeline. */ + queueBuffer(frame, RKISP1_BUFFER_PARAM, paramid); + queueBuffer(frame, RKISP1_BUFFER_STAT, statid); +} + +void IPARkISP1::updateStatistics(unsigned int frame, BufferMemory &statistics) +{ + const rkisp1_stat_buffer *stats = + static_cast(statistics.planes()[0].mem()); + const cifisp_stat *params = &stats->params; + unsigned int aeState = 0; + + if (stats->meas_type & CIFISP_STAT_AUTOEXP) { + const cifisp_ae_stat *ae = ¶ms->ae; + + const unsigned int target = 60; + + unsigned int value = 0; + unsigned int num = 0; + for (int i = 0; i < CIFISP_AE_MEAN_MAX; i++) { + if (ae->exp_mean[i] > 15) { + value += ae->exp_mean[i]; + num++; + } + } + value /= num; + + double factor = (double)target / value; + + if (frame % 3 == 0) { + double tmp; + + tmp = factor * exposure_ * gain_ / minGain_; + exposure_ = utils::clamp((uint64_t)tmp, minExposure_, maxExposure_); + + tmp = tmp / exposure_ * minGain_; + gain_ = utils::clamp((uint64_t)tmp, minGain_, maxGain_); + + setControls(frame + 1); + } + + aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1; + } + + metadataReady(frame, aeState); +} + +void IPARkISP1::setControls(unsigned int frame) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_V4L2_SET; + op.data = { frame }; + + V4L2ControlList ctrls; + ctrls.add(V4L2_CID_EXPOSURE); + ctrls.add(V4L2_CID_ANALOGUE_GAIN); + ctrls[V4L2_CID_EXPOSURE]->value().set(exposure_); + ctrls[V4L2_CID_ANALOGUE_GAIN]->value().set(gain_); + op.v4l2controls.push_back(ctrls); + + queueFrameAction.emit(op); +} + +void IPARkISP1::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_QUEUE_BUFFER; + op.data = { frame, type, id }; + + queueFrameAction.emit(op); +} + +void IPARkISP1::metadataReady(unsigned int frame, unsigned int aeState) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_METADATA; + op.data = { frame, aeState }; + + queueFrameAction.emit(op); +} + +/* + * External IPA module interface + */ + +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 1, + "PipelineHandlerRkISP1", + "RkISP1 IPA", + "LGPL-2.1-or-later", +}; + +IPAInterface *ipaCreate() +{ + return new IPARkISP1(); +} +}; + +}; /* namespace libcamera */ diff --git a/src/ipa/meson.build b/src/ipa/meson.build index 10448c2ffc76af4b..eb5b852eec282735 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -3,6 +3,10 @@ ipa_dummy_sources = [ ['ipa_dummy_isolate', 'Proprietary'], ] +ipa_sources = [ + ['ipa_rkisp1', 'ipa_rkisp1.cpp', 'LGPL-2.1-or-later'], +] + ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources @@ -14,5 +18,14 @@ foreach t : ipa_dummy_sources cpp_args : '-DLICENSE="' + t[1] + '"') endforeach +foreach t : ipa_sources + ipa = shared_module(t[0], t[1], + name_prefix : '', + include_directories : includes, + install : true, + install_dir : ipa_install_dir, + cpp_args : '-DLICENSE="' + t[2] + '"') +endforeach + config_h.set('IPA_MODULE_DIR', '"' + join_paths(get_option('prefix'), ipa_install_dir) + '"') From patchwork Tue Oct 8 00:45:34 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2148 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id DE41061968 for ; Tue, 8 Oct 2019 02:46:50 +0200 (CEST) X-Halon-ID: f889d882-e964-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id f889d882-e964-11e9-837a-0050569116f7; Tue, 08 Oct 2019 02:45:48 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Tue, 8 Oct 2019 02:45:34 +0200 Message-Id: <20191008004534.1585142-11-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> References: <20191008004534.1585142-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 10/10] libcamera: pipeline: rkisp1: Attach to an IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 00:46:51 -0000 Add the plumbing to the pipeline handler to interact with an IPA module. This change makes the usage of an IPA module mandatory for the rkisp1 pipeline. The RkISP1 pipeline handler makes use of a timeline component to schedule actions. This might be useful for other pipeline handlers going forward so keep the generic timeline implementation separate to make it easy to break out. Signed-off-by: Niklas Söderlund --- src/libcamera/pipeline/rkisp1/meson.build | 2 + .../pipeline/rkisp1/rkisp1-timeline.cpp | 51 +++ src/libcamera/pipeline/rkisp1/rkisp1.cpp | 328 +++++++++++++++++- src/libcamera/pipeline/rkisp1/rkisp1.h | 78 +++++ src/libcamera/pipeline/rkisp1/timeline.cpp | 229 ++++++++++++ src/libcamera/pipeline/rkisp1/timeline.h | 72 ++++ 6 files changed, 747 insertions(+), 13 deletions(-) create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1-timeline.cpp create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1.h create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp create mode 100644 src/libcamera/pipeline/rkisp1/timeline.h diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build index f1cc4046b5d064cb..7791e209175d95a9 100644 --- a/src/libcamera/pipeline/rkisp1/meson.build +++ b/src/libcamera/pipeline/rkisp1/meson.build @@ -1,3 +1,5 @@ libcamera_sources += files([ 'rkisp1.cpp', + 'rkisp1-timeline.cpp', + 'timeline.cpp', ]) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1-timeline.cpp b/src/libcamera/pipeline/rkisp1/rkisp1-timeline.cpp new file mode 100644 index 0000000000000000..e6c3f2cc9a09a571 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/rkisp1-timeline.cpp @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1-timeline.cpp - Timeline handler for Rockchip ISP1 + */ + +#include "rkisp1.h" + +#include "log.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(RkISP1) + +void RkISP1ActionSetSensor::run() +{ + sensor_->setControls(&controls_); +} + +void RkISP1ActionQueueBuffer::run() +{ + int ret = device_->queueBuffer(buffer_); + if (ret < 0) + LOG(RkISP1, Error) << "Failed to queue buffer"; +} + +void RkISP1Timeline::bufferReady(Buffer *buffer) +{ + /* + * Calculate SOE by taking the end of DMA set by the kernel and applying + * the time offsets provideprovided by the IPA to find the best estimate + * of SOE. + */ + + ASSERT(frameOffset(SOE) == 0); + + utils::time_point soe = std::chrono::time_point() + + std::chrono::nanoseconds(buffer->timestamp()) + + timeOffset(SOE); + + notifyStartOfExposure(buffer->sequence(), soe); +} + +void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay) +{ + utils::duration delay = std::chrono::milliseconds(msdelay); + setRawDelay(type, frame, delay); +} + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index de4ab523d0e4fe36..74dfc8ca670ffa26 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -5,20 +5,22 @@ * rkisp1.cpp - Pipeline handler for Rockchip ISP1 */ +#include "rkisp1.h" + #include #include #include #include -#include #include #include +#include #include #include -#include "camera_sensor.h" #include "device_enumerator.h" +#include "ipa_manager.h" #include "log.h" #include "media_device.h" #include "pipeline_handler.h" @@ -34,7 +36,7 @@ class RkISP1CameraData : public CameraData { public: RkISP1CameraData(PipelineHandler *pipe) - : CameraData(pipe), sensor_(nullptr) + : CameraData(pipe), sensor_(nullptr), frame_(0) { } @@ -43,8 +45,21 @@ public: delete sensor_; } + int loadIPA(); + Stream stream_; CameraSensor *sensor_; + unsigned int frame_; + std::vector ipaBuffers_; + std::map frameInfo_; + RkISP1Timeline timeline_; + +private: + void queueFrameAction(const IPAOperationData &action); + + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metadataReady(unsigned int frame, unsigned int aeState); }; class RkISP1CameraConfiguration : public CameraConfiguration @@ -99,18 +114,116 @@ private: PipelineHandler::cameraData(camera)); } + friend RkISP1CameraData; + int initLinks(); int createCamera(MediaEntity *sensor); + void tryCompleteRequest(Request *request); void bufferReady(Buffer *buffer); + void paramReady(Buffer *buffer); + void statReady(Buffer *buffer); MediaDevice *media_; V4L2Subdevice *dphy_; V4L2Subdevice *isp_; V4L2VideoDevice *video_; + V4L2VideoDevice *param_; + V4L2VideoDevice *stat_; + + BufferPool paramPool_; + BufferPool statPool_; + + std::map paramBuffers_; + std::map statBuffers_; Camera *activeCamera_; }; +int RkISP1CameraData::loadIPA() +{ + ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->queueFrameAction.connect(this, + &RkISP1CameraData::queueFrameAction); + + return 0; +} + +void RkISP1CameraData::queueFrameAction(const IPAOperationData &action) +{ + switch (action.operation) { + case RKISP1_IPA_ACTION_V4L2_SET: { + unsigned int frame = action.data[0]; + V4L2ControlList controls = action.v4l2controls[0]; + timeline_.scheduleAction(utils::make_unique(frame, + sensor_, + controls)); + break; + } + case RKISP1_IPA_ACTION_QUEUE_BUFFER: { + unsigned int frame = action.data[0]; + unsigned int type = action.data[1]; + unsigned int id = action.data[2]; + queueBuffer(frame, type, id); + break; + } + case RKISP1_IPA_ACTION_METADATA: { + unsigned int frame = action.data[0]; + unsigned aeState = action.data[1]; + metadataReady(frame, aeState); + break; + } + default: + LOG(RkISP1, Error) << "Unkown action " << action.operation; + break; + } +} + +void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + RkISP1ActionType acttype; + V4L2VideoDevice *device; + Buffer *buffer; + switch (type) { + case RKISP1_BUFFER_PARAM: + acttype = QueueParameters; + device = pipe->param_; + buffer = pipe->paramBuffers_[id]; + break; + case RKISP1_BUFFER_STAT: + acttype = QueueStatistics; + device = pipe->stat_; + buffer = pipe->statBuffers_[id]; + break; + default: + LOG(RkISP1, Error) << "Unkown IPA buffer type " << type; + return; + } + + timeline_.scheduleAction(utils::make_unique(frame, + acttype, + device, + buffer)); +} + +void RkISP1CameraData::metadataReady(unsigned int frame, unsigned int aeStatus) +{ + Request *request = frameInfo_[frame]; + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + if (aeStatus) + request->metadata().set(controls::AeLocked, aeStatus == 2); + + pipe->tryCompleteRequest(request); +} + RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera, RkISP1CameraData *data) : CameraConfiguration() @@ -202,12 +315,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) : PipelineHandler(manager), dphy_(nullptr), isp_(nullptr), - video_(nullptr) + video_(nullptr), param_(nullptr), stat_(nullptr) { } PipelineHandlerRkISP1::~PipelineHandlerRkISP1() { + delete param_; + delete stat_; delete video_; delete isp_; delete dphy_; @@ -317,6 +432,20 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) if (ret) return ret; + V4L2DeviceFormat paramFormat = {}; + paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS; + + ret = param_->setFormat(¶mFormat); + if (ret) + return ret; + + V4L2DeviceFormat statFormat = {}; + statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A; + + ret = stat_->setFormat(&statFormat); + if (ret) + return ret; + if (outputFormat.size != cfg.size || outputFormat.fourcc != cfg.pixelFormat) { LOG(RkISP1, Error) @@ -332,39 +461,133 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) int PipelineHandlerRkISP1::allocateBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); Stream *stream = *streams.begin(); + int ret; if (stream->memoryType() == InternalMemory) - return video_->exportBuffers(&stream->bufferPool()); + ret = video_->exportBuffers(&stream->bufferPool()); else - return video_->importBuffers(&stream->bufferPool()); + ret = video_->importBuffers(&stream->bufferPool()); + + if (ret) + return ret; + + paramPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = param_->exportBuffers(¶mPool_); + if (ret) { + video_->releaseBuffers(); + return ret; + } + + statPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = stat_->exportBuffers(&statPool_); + if (ret) { + param_->releaseBuffers(); + video_->releaseBuffers(); + return ret; + } + + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) { + paramBuffers_[i] = new Buffer(i); + statBuffers_[i] = new Buffer(i); + + data->ipaBuffers_.push_back({ + .type = RKISP1_BUFFER_PARAM, + .id = i, + .buffer = paramPool_.buffers()[i], + }); + data->ipaBuffers_.push_back({ + .type = RKISP1_BUFFER_STAT, + .id = i, + .buffer = statPool_.buffers()[i], + }); + } + + data->ipa_->mapBuffers(data->ipaBuffers_); + + return ret; } int PipelineHandlerRkISP1::freeBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); + + data->ipa_->unmapBuffers(data->ipaBuffers_); + data->ipaBuffers_.clear(); + + for (auto it : paramBuffers_) + delete it.second; + + paramBuffers_.clear(); + + for (auto it : statBuffers_) + delete it.second; + + statBuffers_.clear(); + + if (param_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release parameters buffers"; + + if (stat_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release stat buffers"; + if (video_->releaseBuffers()) - LOG(RkISP1, Error) << "Failed to release buffers"; + LOG(RkISP1, Error) << "Failed to release video buffers"; return 0; } int PipelineHandlerRkISP1::start(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; + ret = param_->streamOn(); + if (ret) { + LOG(RkISP1, Error) + << "Failed to start parameters " << camera->name(); + return ret; + } + + ret = stat_->streamOn(); + if (ret) { + param_->streamOff(); + LOG(RkISP1, Error) + << "Failed to start statistics " << camera->name(); + return ret; + } + ret = video_->streamOn(); - if (ret) + if (ret) { + param_->streamOff(); + stat_->streamOff(); + LOG(RkISP1, Error) << "Failed to start camera " << camera->name(); + } activeCamera_ = camera; + /* Inform IPA of stream configuration and sensor controls. */ + std::map streamConfig; + streamConfig[0] = { + .pixelFormat = data->stream_.configuration().pixelFormat, + .size = data->stream_.configuration().size, + }; + + std::map entityControls; + entityControls[0] = data->sensor_->controls(); + + data->ipa_->configure(streamConfig, entityControls); + return ret; } void PipelineHandlerRkISP1::stop(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; ret = video_->streamOff(); @@ -372,6 +595,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera) LOG(RkISP1, Warning) << "Failed to stop camera " << camera->name(); + ret = stat_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop statistics " << camera->name(); + + ret = param_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop parameters " << camera->name(); + + data->timeline_.reset(); + activeCamera_ = nullptr; } @@ -387,12 +622,22 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request) return -ENOENT; } - int ret = video_->queueBuffer(buffer); - if (ret < 0) - return ret; - PipelineHandler::queueRequest(camera, request); + data->frameInfo_[data->frame_] = request; + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST; + op.data = { data->frame_ }; + op.controls = { request->controls() }; + data->ipa_->processEvent(op); + + data->timeline_.scheduleAction(utils::make_unique(data->frame_, + QueueVideo, + video_, + buffer)); + data->frame_++; + return 0; } @@ -435,11 +680,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) std::unique_ptr data = utils::make_unique(this); + data->controlInfo_.emplace(std::piecewise_construct, + std::forward_as_tuple(&controls::AeEnable), + std::forward_as_tuple(false, true)); + data->sensor_ = new CameraSensor(sensor); ret = data->sensor_->init(); if (ret) return ret; + ret = data->loadIPA(); + if (ret) + return ret; + std::set streams{ &data->stream_ }; std::shared_ptr camera = Camera::create(this, sensor->name(), streams); @@ -478,7 +731,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) if (video_->open() < 0) return false; + stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics"); + if (stat_->open() < 0) + return false; + + param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params"); + if (param_->open() < 0) + return false; + video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady); + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); /* Configure default links. */ if (initLinks() < 0) { @@ -504,13 +767,52 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) * Buffer Handling */ +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request) +{ + if (request->hasPendingBuffers()) + return; + + if (request->metadata().empty()) + return; + + completeRequest(activeCamera_, request); +} + void PipelineHandlerRkISP1::bufferReady(Buffer *buffer) { ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); Request *request = buffer->request(); + data->timeline_.bufferReady(buffer); + + if (data->frame_ <= buffer->sequence()) + data->frame_ = buffer->sequence() + 1; + completeBuffer(activeCamera_, request, buffer); - completeRequest(activeCamera_, request); + tryCompleteRequest(request); +} + +void PipelineHandlerRkISP1::paramReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_PARAM, buffer->index() }; + data->ipa_->processEvent(op); +} + +void PipelineHandlerRkISP1::statReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_STAT, buffer->index() }; + data->ipa_->processEvent(op); } REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1); diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h new file mode 100644 index 0000000000000000..f9ea3a822cb1d0f3 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/rkisp1.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Pipeline handler for Rockchip ISP1 + */ +#ifndef __LIBCAMERA_RKISP1_H__ +#define __LIBCAMERA_RKISP1_H__ + +#include +#include + +#include "camera_sensor.h" +#include "timeline.h" +#include "v4l2_videodevice.h" + +namespace libcamera { + +enum RkISP1ActionType { + SetSensor, + SOE, + QueueVideo, + QueueParameters, + QueueStatistics, +}; + +class RkISP1ActionSetSensor : public FrameAction +{ +public: + RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls) + : FrameAction(frame, SetSensor), sensor_(sensor), controls_(controls) {} + +protected: + void run() override; + +private: + CameraSensor *sensor_; + V4L2ControlList controls_; +}; + +class RkISP1ActionQueueBuffer : public FrameAction +{ +public: + RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type, + V4L2VideoDevice *device, Buffer *buffer) + : FrameAction(frame, type), device_(device), buffer_(buffer) + { + } + +protected: + void run() override; + +private: + V4L2VideoDevice *device_; + Buffer *buffer_; +}; + +class RkISP1Timeline : public Timeline +{ +public: + RkISP1Timeline() + : Timeline() + { + setDelay(SetSensor, -1, 5); + setDelay(SOE, 0, -1); + setDelay(QueueVideo, -1, 10); + setDelay(QueueParameters, -1, 8); + setDelay(QueueStatistics, -1, 8); + } + + void bufferReady(Buffer *buffer); + + void setDelay(unsigned int type, int frame, int msdelay); +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_RKISP1_H__ */ diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp new file mode 100644 index 0000000000000000..95ec6b7ca0a0237a --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline for per-frame control + */ + +#include "timeline.h" + +#include "log.h" + +/** + * \file timeline.h + * \brief Timeline for per-frame control + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Timeline) + +/** + * \class FrameAction + * \brief Action that can be schedule on a Timeline + * + * A frame action is an event that can be schedule to be executed at a point + * in time. A frame action has two primal attributes: frame number and type. + * + * The frame number describes the frame for which the action should have an + * effect. The type is a numerical ID which identifies the action within the + * pipeline and IPA protocol. + * + * \sa Timeline + */ + +/** + * \class Timeline + * \brief Executor of FrameActions + * + * The timeline have three primary functions. + * + * 1. Keep track of the Start of Exposure (SOE) for every frame processed by + * the hardware. Using this information it shall keep an up-to-date estimate + * of the frame interval (time between two SOEs). + * + * The estimated frame interval together with recorded SOE events is the + * foundation for how the timeline schedule FrameActions at specific points + * in time. + * \todo Improve the frame interval estimation algorithm. + * + * 2. Keep track of current delays for different types of actions. The delays + * for different actions might differ during a capture session. Exposure time + * effects the over all FPS and different ISP parameters might impacts its + * processing time. + * + * The action type delays shall be updated by the IPA in conjunction with + * how it changes the capture parameters. + * + * 3. Schedule actions on the timeline. This is the process of taking a + * FrameAction which contains an abstract description of what frame and + * what type of action it contains and turning that into an time point + * and make sure the action is executed at that time. + */ + +Timeline::Timeline() + : frameInterval_(0) +{ + timer_.timeout.connect(this, &Timeline::timeout); +} + +/** + * \brief Reset and stop the timeline + * + * The timeline needs to be reset when the timeline should no longer execute + * actions. A timeline should be reset between two capture sessions to prevent + * the old capture session to effect the second one. + */ +void Timeline::reset() +{ + timer_.stop(); + + actions_.clear(); + history_.clear(); +} + +/** + * \brief Schedule action on the timeline + * \param[in] action FrameAction to schedule + * + * The act of scheduling an action to the timeline is the process of taking + * the properties of the action (type, frame and time offsets) and translating + * that to a time point using the current values for the action type timings + * value recorded in the timeline. If an action is scheduled too late, execute + * it immediately. + */ +void Timeline::scheduleAction(std::unique_ptr action) +{ + unsigned int lastFrame; + utils::time_point lastTime; + + if (history_.empty()) { + lastFrame = 0; + lastTime = std::chrono::steady_clock::now(); + } else { + lastFrame = history_.back().first; + lastTime = history_.back().second; + } + + /* + * Calculate when the action shall be schedule by first finding out how + * many frames in the future the action acts on and then add the actions + * frame offset. After the spatial frame offset is found out translate + * that to a time point by using the last estimated start of exposure + * (SOE) as the fixed offset. Lastly add the actions time offset to the + * time point. + */ + int frame = action->frame() - lastFrame + frameOffset(action->type()); + utils::time_point deadline = lastTime + frame * frameInterval_ + + timeOffset(action->type()); + + utils::time_point now = std::chrono::steady_clock::now(); + if (deadline < now) { + LOG(Timeline, Warning) + << "Action scheduled too late " + << utils::time_point_to_string(deadline) + << ", run now " << utils::time_point_to_string(now); + action->run(); + } else { + actions_.insert({ deadline, std::move(action) }); + updateDeadline(); + } +} + +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time) +{ + history_.push_back(std::make_pair(frame, time)); + + if (history_.size() <= HISTORY_DEPTH / 2) + return; + + while (history_.size() > HISTORY_DEPTH) + history_.pop_front(); + + /* Update esitmated time between two start of exposures. */ + utils::duration sumExposures(0); + unsigned int numExposures = 0; + + utils::time_point lastTime; + for (auto it = history_.begin(); it != history_.end(); it++) { + if (it != history_.begin()) { + sumExposures += it->second - lastTime; + numExposures++; + } + + lastTime = it->second; + } + + frameInterval_ = sumExposures; + if (numExposures) + frameInterval_ /= numExposures; +} + +int Timeline::frameOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No frame offset set for action type " << type; + return 0; + } + + return it->second.first; +} + +utils::duration Timeline::timeOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No time offset set for action type " << type; + return utils::duration::zero(); + } + + return it->second.second; +} + +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time) +{ + delays_[type] = std::make_pair(frame, time); +} + +void Timeline::updateDeadline() +{ + if (actions_.empty()) + return; + + const utils::time_point &deadline = actions_.begin()->first; + + if (timer_.isRunning() && deadline >= timer_.deadline()) + return; + + if (deadline <= std::chrono::steady_clock::now()) { + timeout(&timer_); + return; + } + + timer_.start(deadline); +} + +void Timeline::timeout(Timer *timer) +{ + utils::time_point now = std::chrono::steady_clock::now(); + + for (auto it = actions_.begin(); it != actions_.end();) { + const utils::time_point &sched = it->first; + + if (sched > now) + break; + + FrameAction *action = it->second.get(); + + action->run(); + + it = actions_.erase(it); + } + + updateDeadline(); +} + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/rkisp1/timeline.h b/src/libcamera/pipeline/rkisp1/timeline.h new file mode 100644 index 0000000000000000..9d30e4eaf8743d07 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/timeline.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.h - Timeline for per-frame controls + */ +#ifndef __LIBCAMERA_TIMELINE_H__ +#define __LIBCAMERA_TIMELINE_H__ + +#include +#include + +#include + +#include "utils.h" + +namespace libcamera { + +class FrameAction +{ +public: + FrameAction(unsigned int frame, unsigned int type) + : frame_(frame), type_(type) {} + + virtual ~FrameAction() {} + + unsigned int frame() const { return frame_; } + unsigned int type() const { return type_; } + + virtual void run() = 0; + +private: + unsigned int frame_; + unsigned int type_; +}; + +class Timeline +{ +public: + Timeline(); + virtual ~Timeline() {} + + virtual void reset(); + virtual void scheduleAction(std::unique_ptr action); + virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time); + + utils::duration frameInterval() const { return frameInterval_; } + +protected: + int frameOffset(unsigned int type) const; + utils::duration timeOffset(unsigned int type) const; + + void setRawDelay(unsigned int type, int frame, utils::duration time); + + std::map> delays_; + +private: + static constexpr unsigned int HISTORY_DEPTH = 10; + + void timeout(Timer *timer); + void updateDeadline(); + + std::list> history_; + std::multimap> actions_; + utils::duration frameInterval_; + + Timer timer_; +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_TIMELINE_H__ */