From patchwork Thu Oct 3 17:49:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2082 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 8A8C36165B for ; Thu, 3 Oct 2019 19:50:37 +0200 (CEST) X-Halon-ID: 3138de1e-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 3138de1e-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:48 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:31 +0200 Message-Id: <20191003174941.1296988-2-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 01/11] libcamera: pipeline: Move IPA from pipeline to camera data X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:37 -0000 The IPA acts on a camera and not on a pipeline which can expose more then one camera. Move the IPA reference to the CameraData. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/include/pipeline_handler.h | 2 ++ src/libcamera/pipeline/vimc.cpp | 12 +++++------- src/libcamera/pipeline_handler.cpp | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/libcamera/include/pipeline_handler.h b/src/libcamera/include/pipeline_handler.h index 1fdef9cea40f1f0a..42b90a4bf1a6e414 100644 --- a/src/libcamera/include/pipeline_handler.h +++ b/src/libcamera/include/pipeline_handler.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ public: PipelineHandler *pipe_; std::list queuedRequests_; ControlInfoMap controlInfo_; + std::unique_ptr ipa_; private: CameraData(const CameraData &) = delete; diff --git a/src/libcamera/pipeline/vimc.cpp b/src/libcamera/pipeline/vimc.cpp index 26477dccbb8fcd5b..f1cfd0ed35cfd9cd 100644 --- a/src/libcamera/pipeline/vimc.cpp +++ b/src/libcamera/pipeline/vimc.cpp @@ -101,8 +101,6 @@ private: return static_cast( PipelineHandler::cameraData(camera)); } - - std::unique_ptr ipa_; }; VimcCameraConfiguration::VimcCameraConfiguration() @@ -353,13 +351,13 @@ bool PipelineHandlerVimc::match(DeviceEnumerator *enumerator) if (!media) return false; - ipa_ = IPAManager::instance()->createIPA(this, 0, 0); - if (ipa_ == nullptr) + std::unique_ptr data = utils::make_unique(this); + + data->ipa_ = IPAManager::instance()->createIPA(this, 0, 0); + if (data->ipa_ == nullptr) LOG(VIMC, Warning) << "no matching IPA found"; else - ipa_->init(); - - std::unique_ptr data = utils::make_unique(this); + data->ipa_->init(); /* Locate and open the capture video node. */ if (data->init(media)) diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index 3e54aa23d92b9a36..2d69dea843dd0980 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -96,6 +96,14 @@ LOG_DEFINE_CATEGORY(Pipeline) * creating the camera, and shall not be modified afterwards. */ +/** + * \var CameraData::ipa_ + * \brief The IPA module used by the camera + * + * Reference to the Image Processing Algorithms (IPA) operating on the camera's + * stream(s). If no IPAs are in operation this should be set to nullptr. + */ + /** * \class PipelineHandler * \brief Create and manage cameras based on a set of media devices From patchwork Thu Oct 3 17:49:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2084 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id E5E7F616FF for ; Thu, 3 Oct 2019 19:50:37 +0200 (CEST) X-Halon-ID: 318adaa0-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 318adaa0-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:48 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:32 +0200 Message-Id: <20191003174941.1296988-3-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 02/11] libcamera: controls: Add AeEnable control X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:38 -0000 Add a control to enable/disable Auto Exposure. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/control_ids.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcamera/control_ids.yaml b/src/libcamera/control_ids.yaml index 819a5967a2fc8d56..4626f6ef48bcaeea 100644 --- a/src/libcamera/control_ids.yaml +++ b/src/libcamera/control_ids.yaml @@ -5,6 +5,13 @@ %YAML 1.2 --- controls: + - AeEnable: + type: bool + description: | + Enables or disables the AE. + + \sa ManualExposure + - AwbEnable: type: bool description: | From patchwork Thu Oct 3 17:49:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2085 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 2812E60BE9 for ; Thu, 3 Oct 2019 19:50:38 +0200 (CEST) X-Halon-ID: 31cc35b6-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 31cc35b6-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:49 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:33 +0200 Message-Id: <20191003174941.1296988-4-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 03/11] libcamera: controls: Add AeLocked control X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:38 -0000 Add a control to report lock status of the Auto Exposure algorithm. The idea is that if an AE algorithm is running the control shall be added to the meta data control list. If the AE algorithm is locked the value shall be set to true, if it's converging it shall be set to false. If the AE algorithm is not running the control shall not be present in the meta data control list. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- src/libcamera/control_ids.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcamera/control_ids.yaml b/src/libcamera/control_ids.yaml index 4626f6ef48bcaeea..142bdede2b96c0e0 100644 --- a/src/libcamera/control_ids.yaml +++ b/src/libcamera/control_ids.yaml @@ -12,6 +12,13 @@ controls: \sa ManualExposure + - AeLocked: + type: bool + description: | + Report the lock status of a running AE algorithm. + + \sa AeEnable + - AwbEnable: type: bool description: | From patchwork Thu Oct 3 17:49:34 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2086 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 772EB61915 for ; Thu, 3 Oct 2019 19:50:38 +0200 (CEST) X-Halon-ID: 320eb6db-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 320eb6db-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:49 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:34 +0200 Message-Id: <20191003174941.1296988-5-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 04/11] libcamera: request: Add meta data information X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:38 -0000 A new ControlList container is needed to hold meta data coming out of the IPA. The list of supported controls in this list is expected to grow and for now do not add a validator for the list. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- include/libcamera/request.h | 2 ++ src/libcamera/request.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/libcamera/request.h b/include/libcamera/request.h index e3db5243aaf3cf30..d3435d134c560f44 100644 --- a/include/libcamera/request.h +++ b/include/libcamera/request.h @@ -37,6 +37,7 @@ public: ~Request(); ControlList &controls() { return *controls_; } + ControlList &metaData() { return *metaData_; } const std::map &buffers() const { return bufferMap_; } int addBuffer(std::unique_ptr buffer); Buffer *findBuffer(Stream *stream) const; @@ -58,6 +59,7 @@ private: Camera *camera_; CameraControlValidator *validator_; ControlList *controls_; + ControlList *metaData_; std::map bufferMap_; std::unordered_set pending_; diff --git a/src/libcamera/request.cpp b/src/libcamera/request.cpp index 19f6d0b9a0aeb03c..1e1418a9b57393b3 100644 --- a/src/libcamera/request.cpp +++ b/src/libcamera/request.cpp @@ -65,6 +65,11 @@ Request::Request(Camera *camera, uint64_t cookie) */ validator_ = new CameraControlValidator(camera); controls_ = new ControlList(validator_); + + /** + * \todo: Add a validator for meta data controls. + */ + metaData_ = new ControlList(nullptr); } Request::~Request() @@ -74,6 +79,7 @@ Request::~Request() delete buffer; } + delete metaData_; delete controls_; delete validator_; } @@ -161,6 +167,12 @@ Buffer *Request::findBuffer(Stream *stream) const return it->second; } +/** + * \fn Request::metaData() + * \brief Retrieve the request's meta data + * \return The meta data associated with the request + */ + /** * \fn Request::cookie() * \brief Retrieve the cookie set when the request was created From patchwork Thu Oct 3 17:49:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2088 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 05D606165B for ; Thu, 3 Oct 2019 19:50:39 +0200 (CEST) X-Halon-ID: 325ae7ce-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 325ae7ce-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:50 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:35 +0200 Message-Id: <20191003174941.1296988-6-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 05/11] libcamera: ipa: meson: Allow access to internal libcamera headers X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:40 -0000 Allow IPA implementations to use internal libcamera headers so they can utilize helpers not exposed to applications. Signed-off-by: Niklas Söderlund Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/ipa/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipa/meson.build b/src/ipa/meson.build index f09915bc13887bd3..10448c2ffc76af4b 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -8,7 +8,7 @@ ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources ipa = shared_module(t[0], 'ipa_dummy.cpp', name_prefix : '', - include_directories : libcamera_includes, + include_directories : includes, install : true, install_dir : ipa_install_dir, cpp_args : '-DLICENSE="' + t[1] + '"') From patchwork Thu Oct 3 17:49:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2087 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 17BFE6170D for ; Thu, 3 Oct 2019 19:50:40 +0200 (CEST) X-Halon-ID: 32b2c12b-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 32b2c12b-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:50 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:36 +0200 Message-Id: <20191003174941.1296988-7-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 06/11] libcamera: ipa: Extend to support IPA interactions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:40 -0000 The IPA interface needs to support interactions with the pipeline, add interfaces to control the sensor and handling of request ISP parameters and statistics. Signed-off-by: Niklas Söderlund --- include/ipa/ipa_interface.h | 37 +++++++ src/ipa/ipa_dummy.cpp | 7 +- src/libcamera/ipa_interface.cpp | 132 ++++++++++++++++++++++++ src/libcamera/proxy/ipa_proxy_linux.cpp | 14 ++- 4 files changed, 181 insertions(+), 9 deletions(-) diff --git a/include/ipa/ipa_interface.h b/include/ipa/ipa_interface.h index 2c5eb1fd524311cb..2c715314c7a418f0 100644 --- a/include/ipa/ipa_interface.h +++ b/include/ipa/ipa_interface.h @@ -7,14 +7,51 @@ #ifndef __LIBCAMERA_IPA_INTERFACE_H__ #define __LIBCAMERA_IPA_INTERFACE_H__ +#include +#include + +#include +#include +#include +#include + +#include "v4l2_controls.h" + namespace libcamera { +struct IPAStream { + unsigned int pixelFormat; + Size size; +}; + +struct IPABuffer { + unsigned int id; + unsigned int type; + BufferMemory buffer; +}; + +struct IPAOperationData { + unsigned int operation; + std::vector data; + std::vector controls; + /* \todo: Add more vectors for data types used in pipa<->IPA interactions. */ +}; + class IPAInterface { public: virtual ~IPAInterface() {} virtual int init() = 0; + + virtual void configure(const std::map &streamConfig, + const std::map &entityControls) = 0; + + virtual void mapBuffers(const std::vector &buffers) = 0; + virtual void unmapBuffers(const std::vector &buffers) = 0; + + virtual void processEvent(const IPAOperationData &event) = 0; + Signal queueFrameAction; }; } /* namespace libcamera */ diff --git a/src/ipa/ipa_dummy.cpp b/src/ipa/ipa_dummy.cpp index 9d0cbdc8b1ad5565..08f74c4952f3923a 100644 --- a/src/ipa/ipa_dummy.cpp +++ b/src/ipa/ipa_dummy.cpp @@ -15,7 +15,12 @@ namespace libcamera { class IPADummy : public IPAInterface { public: - int init(); + int init() override; + void configure(const std::map &streamConfig, + const std::map &entityControls) override {} + void mapBuffers(const std::vector &buffers) override {} + void unmapBuffers(const std::vector &buffers) override {} + void processEvent(const IPAOperationData &event) override {} }; int IPADummy::init() diff --git a/src/libcamera/ipa_interface.cpp b/src/libcamera/ipa_interface.cpp index d7d8ca8881efcf66..c0f2004f3ec993b2 100644 --- a/src/libcamera/ipa_interface.cpp +++ b/src/libcamera/ipa_interface.cpp @@ -14,6 +14,68 @@ namespace libcamera { + +/** + * \struct IPAStream + * \brief Hold IPA description of a stream. + */ + +/** + * \var IPAStream::pixelFormat + * \brief The streams pixel format + */ + +/** + * \var IPAStream::size + * \brief The streams size + */ + +/** + * \struct IPABuffer + * \brief Hold IPA description of a buffer + */ + +/** + * \var IPABuffer::id + * \brief The pipeline to IPA id cookie for the buffer + */ + +/** + * \var IPABuffer::type + * \brief The pipeline to IPA type for the buffer + */ + +/** + * \var IPABuffer::buffer + * \brief The hardware description of the buffer shared between pipeline and IPA + */ + +/** + * \struct IPAOperationData + * \brief Hold data exchanged between pipeline and IPA + * + * This is the information carrier between pipeline and IPA. The is is used + * to transport the pipeline specific protocol between the two. Core libcamera + * components do not try to interpret the protocol and it's up to the pipeline + * handler to uses the members of this struct in any way they see fit, and to + * document it so multiple IPA implementations for the same pipeline may use it. + */ + +/** + * \var IPAOperationData::operation + * \brief Intended as the operation code in the pipeline to IPA protocol + */ + +/** + * \var IPAOperationData::data + * \brief Intended as the generic data carrier in the pipeline to IPA protocol + */ + +/** + * \var IPAOperationData::controls + * \brief Intended as the ControlList carrier in the pipeline to IPA protocol + */ + /** * \class IPAInterface * \brief Interface for IPA implementation @@ -24,4 +86,74 @@ namespace libcamera { * \brief Initialise the IPAInterface */ +/** + * \fn IPAInterface::configure() + * \brief Configure the IPA stream and sensor settings + * \param[in] streamConfig List of stream configuration descriptions + * \param[in] entityControls List of controls provided by the pipeline entities + * + * This function is called when a pipeline attaches to an IPA to inform the IPA + * of the streams and controls limits the entities(s) in the video pipeline + * supports. + */ + +/** + * \fn IPAInterface::mapBuffers() + * \brief Map the buffers shared by the pipeline to the IPA + * \param[in] buffers List of buffers to map + * + * This function is called when a pipeline handler wants to inform the IPA of + * which buffers it has mapped which the IPA can make use of. All buffers shared + * between these two object's needs to be shared using this function prior to + * use. + * + * After the buffers have been initialized a specific buffer can be referenced + * using the numerical \a type and \a id provided when the buffers where mapped. + * + * The numerical values for type and id used to describe a buffer have no + * meaning in the core libcamera code and the interface is pipeline handler + * specific. + * + * \sa unmapBuffers() + * \todo Can we make this a generic function? + */ + +/** + * \fn IPAInterface::unmapBuffers() + * \brief Unmap the buffers shared by the pipeline to the IPA + * \param[in] buffers List of buffers to unmap + * + * This function is called when a pipeline handler wants to the IPA to unmap + * all or some of the buffer it have mapped. + * + * \sa mapBuffers() + * \todo Can we make this a generic function? + */ + +/** + * \fn IPAInterface::processEvent() + * \brief Process an event from the pipeline handler + * \param[in] event Event to process + * + * The pipeline handler can send events to the IPA to notify it of what is + * going on. This is the entry point on the IPA side for those messages. + * + * The protocol between pipeline and IPA are hilly specific for each pipeline + * and needs to be documented separately. Libcamera reserves no operation + * numbers and make no attempt to interpret the protocol. + */ + +/** + * \fn IPAInterface::queueFrameAction() + * \brief Signal emitted when the IPA wish to queue a FrameAction + * \param[in] frame The frame number for the request + * \param[in] controls List of controls associated with the request + * + * This signal is emitted when the IPA wish to queue a FrameAction on the + * pipeline. The pipeline is still responsible for the scheduling of the action + * on its timeline. + * + * The IPA operation describing the frame action is passed as a parameter. + */ + } /* namespace libcamera */ diff --git a/src/libcamera/proxy/ipa_proxy_linux.cpp b/src/libcamera/proxy/ipa_proxy_linux.cpp index 62fcb529e1c7e4ff..14e8bb6067623fc6 100644 --- a/src/libcamera/proxy/ipa_proxy_linux.cpp +++ b/src/libcamera/proxy/ipa_proxy_linux.cpp @@ -26,7 +26,12 @@ public: IPAProxyLinux(IPAModule *ipam); ~IPAProxyLinux(); - int init(); + int init() override { return 0; } + void configure(const std::map &streamConfig, + const std::map &entityControls) override {} + void mapBuffers(const std::vector &buffers) override {} + void unmapBuffers(const std::vector &buffers) override {} + void processEvent(const IPAOperationData &event) override {} private: void readyRead(IPCUnixSocket *ipc); @@ -36,13 +41,6 @@ private: IPCUnixSocket *socket_; }; -int IPAProxyLinux::init() -{ - LOG(IPAProxy, Debug) << "initializing IPA via dummy proxy!"; - - return 0; -} - IPAProxyLinux::IPAProxyLinux(IPAModule *ipam) : proc_(nullptr), socket_(nullptr) { From patchwork Thu Oct 3 17:49:37 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2089 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D20176165B for ; Thu, 3 Oct 2019 19:50:40 +0200 (CEST) X-Halon-ID: 33267621-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 33267621-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:51 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:37 +0200 Message-Id: <20191003174941.1296988-8-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 07/11] libcamera: timeline: Add a basic timeline implementation X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:41 -0000 The timeline is a helper for pipeline handlers to ease interacting with an IPA. The idea is that the pipeline handler runs a timeline which schedules and executes actions on the hardware. The pipeline listens to signals from the IPA which it translates into actions which are schedule on the timeline. Signed-off-by: Niklas Söderlund --- src/libcamera/include/meson.build | 1 + src/libcamera/include/timeline.h | 71 ++++++++ src/libcamera/meson.build | 1 + src/libcamera/timeline.cpp | 267 ++++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 src/libcamera/include/timeline.h create mode 100644 src/libcamera/timeline.cpp diff --git a/src/libcamera/include/meson.build b/src/libcamera/include/meson.build index 2c74d29bd925ede2..238ddc2c2cf96a5d 100644 --- a/src/libcamera/include/meson.build +++ b/src/libcamera/include/meson.build @@ -18,6 +18,7 @@ libcamera_headers = files([ 'pipeline_handler.h', 'process.h', 'thread.h', + 'timeline.h', 'utils.h', 'v4l2_controls.h', 'v4l2_device.h', diff --git a/src/libcamera/include/timeline.h b/src/libcamera/include/timeline.h new file mode 100644 index 0000000000000000..1ec28a0d9134d35c --- /dev/null +++ b/src/libcamera/include/timeline.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.h - Timeline for per-frame controls + */ +#ifndef __LIBCAMERA_TIMELINE_H__ +#define __LIBCAMERA_TIMELINE_H__ + +#include +#include + +#include + +#include "utils.h" + +namespace libcamera { + +class FrameAction +{ +public: + FrameAction(unsigned int type, unsigned int frame) + : type_(type), frame_(frame) {} + + virtual ~FrameAction() {} + + unsigned int type() const { return type_; } + unsigned int frame() const { return frame_; } + + virtual void run() = 0; + +private: + unsigned int type_; + unsigned int frame_; +}; + +class Timeline +{ +public: + Timeline(); + virtual ~Timeline() {} + + virtual void reset(); + virtual void scheduleAction(FrameAction *action); + virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time); + + utils::duration frameInterval() const { return frameInterval_; } + +protected: + int frameOffset(unsigned int type) const; + utils::duration timeOffset(unsigned int type) const; + + void setRawDelay(unsigned int type, int frame, utils::duration time); + +private: + static constexpr unsigned int HistoryDepth = 10; + + void timeout(Timer *timer); + void updateDeadline(const utils::time_point &deadline); + + std::list> history_; + std::multimap actions_; + std::map> delays_; + utils::duration frameInterval_; + + Timer timer_; +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_TIMELINE_H__ */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 3a3b388a6a344961..195368e77e5e6f01 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -30,6 +30,7 @@ libcamera_sources = files([ 'signal.cpp', 'stream.cpp', 'thread.cpp', + 'timeline.cpp', 'timer.cpp', 'utils.cpp', 'v4l2_controls.cpp', diff --git a/src/libcamera/timeline.cpp b/src/libcamera/timeline.cpp new file mode 100644 index 0000000000000000..042ead41d2ff54d5 --- /dev/null +++ b/src/libcamera/timeline.cpp @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline for per-frame controls + */ + +#include "timeline.h" + +#include "log.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Timeline) + +/** + * \class FrameAction + * \brief Action which can be schedule within a frame's lifetime + * + * A frame action is an event which takes place at some point during a frame's + * lifetime inside libcamera. Each action has two primal attributes; type and + * frame number. + * + * The type is a numerical ID which identifies the action within the pipeline + * handler. The type is pipeline specific and has no meaning outside the + * specific implementation. The frame number is the frame the action applies to. + */ + +/** + * \fn FrameAction::FrameAction + * \brief Create a frame action + * \param[in] type Action type + * \param[in] frame Frame number + */ + +/** + * \fn FrameAction::type() + * \brief Retrieve the type of action + * \return Action type + */ + +/** + * \fn FrameAction::frame() + * \brief Retrieve the frame number + * \return Frame number + */ + +/** + * \fn FrameAction::run() + * \brief The action to perform when the action is triggered + * + * Pipeline handlers shall subclass the FrameAction object and implement run() + * functions which describes the actions they wish to happen when the act is + * schedule using the Timeline. + * + * \sa Timeline + */ + +/** + * \class Timeline + * \brief Scheduler and executor of FrameAction's + * + * On the timeline the pipeline can schedule FrameAction and expect them to be + * executed at the correct time. The correct time to execute them is a sum + * of which frame the action should apply to and a list of timing delays for + * each action the pipeline handler describes. + * + * The timing delays can either be set by the pipeline handler or by the IPA. + * The exact implementation of how the timing delays are setup is pipeline + * specific. It is expected that the IPA will update the timing delays as it + * make changes to how the pipeline operates in different situations. + * + * The pipeline handler is responsible for feeding the timeline as accurate as + * possible information of the exposures it observes. + */ + +Timeline::Timeline() + : frameInterval_(std::chrono::milliseconds(1)) +{ + timer_.timeout.connect(this, &Timeline::timeout); +} + +/** + * \brief Reset the timeline + */ +void Timeline::reset() +{ + timer_.stop(); + + actions_.clear(); + history_.clear(); +} + +/** + * \brief Add a action to the timeline + * \param[in] action FrameAction to add + * + * Schedule an action at a specific time taking the action and timing delays + * into account. If a action is schedule too late execute it immediately and + * try to recover. + */ +void Timeline::scheduleAction(FrameAction *action) +{ + unsigned int lastFrame; + utils::time_point lastTime; + + if (history_.empty()) { + /* + * This only happens for the first number of frames, up to + * pipeline depth. + */ + lastFrame = 0; + lastTime = std::chrono::steady_clock::now(); + } else { + lastFrame = history_.back().first; + lastTime = history_.back().second; + } + + /* + * Calculate action trigger time by first figuring the out the start of + * exposure (SOE) of the frame the action corresponds to and then adding + * the frame and timing offsets. + */ + int frame = action->frame() + frameOffset(action->type()) - lastFrame; + utils::time_point deadline = lastTime + frame * frameInterval_ + + timeOffset(action->type()); + + utils::time_point now = std::chrono::steady_clock::now(); + if (deadline < now) { + LOG(Timeline, Warning) + << "Action scheduled too late " + << utils::time_point_to_string(deadline) + << ", run now " << utils::time_point_to_string(now); + action->run(); + } else { + actions_.insert({ deadline, action }); + updateDeadline(deadline); + } +} + +/** + * \brief Inform timeline of a new exposure + * \param[in] frame Frame number of the exposure + * \param[in] time The best approximation of when the exposure started + */ +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time) +{ + history_.push_back(std::make_pair(frame, time)); + + if (history_.size() <= HistoryDepth / 2) + return; + + while (history_.size() > HistoryDepth) + history_.pop_front(); + + /* Update esitmated time between two start of exposures. */ + utils::duration sumExposures = std::chrono::duration_values::zero(); + unsigned int numExposures = 0; + + utils::time_point lastTime; + for (auto it = history_.begin(); it != history_.end(); it++) { + if (it != history_.begin()) { + sumExposures += it->second - lastTime; + numExposures++; + } + + lastTime = it->second; + } + + frameInterval_ = sumExposures; + if (numExposures) + frameInterval_ /= numExposures; +} + +/** + * \fn Timeline::frameInterval() + * \brief Retrieve the best estimate of the frame interval + * \return Frame interval estimate + */ + +/** + * \brief Retrieve the frame offset for an action type + * \param[in] type Action type + * \return Frame offset for the action type + */ +int Timeline::frameOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No frame offset set for action type " << type; + return 0; + } + + return it->second.first; +} + +/** + * \brief Retrieve the time offset for an action type + * \param[in] type Action type + * \return Time offset for the action type + */ +utils::duration Timeline::timeOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No time offset set for action type " << type; + return utils::duration::zero(); + } + + return it->second.second; +} + +/** + * \brief Set the frame and time offset delays for an action type + * \param[in] type Action type + * \param[in] frame Frame offset + * \param[in] time Time offset + */ +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time) +{ + delays_[type] = std::make_pair(frame, time); +} + +void Timeline::updateDeadline(const utils::time_point &deadline) +{ + if (timer_.isRunning() && deadline >= timer_.deadline()) + return; + + utils::time_point now = std::chrono::steady_clock::now(); + utils::duration duration = deadline - now; + + /* + * Translate nanoseconds resolution the timeline to milliseconds using + * a ceiling approach to not trigger an action before it's scheduled. + * + * \todo: Should the Timer operate using nanoseconds? + */ + unsigned long nsecs = std::chrono::duration_cast(duration).count(); + unsigned int msecs = nsecs / 1000000 + (nsecs % 1000000 ? 1 : 0); + + timer_.stop(); + timer_.start(msecs); +} + +void Timeline::timeout(Timer *timer) +{ + for (auto it = actions_.begin(); it != actions_.end();) { + utils::time_point now = std::chrono::steady_clock::now(); + + utils::time_point sched = it->first; + FrameAction *action = it->second; + + if (sched >= now) { + updateDeadline(sched); + break; + } + + action->run(); + + it = actions_.erase(it); + delete action; + } +} + +} /* namespace libcamera */ From patchwork Thu Oct 3 17:49:38 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2090 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4DB2A61925 for ; Thu, 3 Oct 2019 19:50:41 +0200 (CEST) X-Halon-ID: 33b48f9b-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 33b48f9b-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:52 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:38 +0200 Message-Id: <20191003174941.1296988-9-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 08/11] test: Add timeline test X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:41 -0000 Add a test of the timeline to make sure events fire and the estimated frame interval are OK. Signed-off-by: Niklas Söderlund --- test/meson.build | 1 + test/timeline.cpp | 97 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 test/timeline.cpp diff --git a/test/meson.build b/test/meson.build index 84722cceb35db86e..6a4c4cd373043469 100644 --- a/test/meson.build +++ b/test/meson.build @@ -28,6 +28,7 @@ internal_tests = [ ['object-invoke', 'object-invoke.cpp'], ['signal-threads', 'signal-threads.cpp'], ['threads', 'threads.cpp'], + ['timeline', 'timeline.cpp'], ['timer', 'timer.cpp'], ['timer-thread', 'timer-thread.cpp'], ] diff --git a/test/timeline.cpp b/test/timeline.cpp new file mode 100644 index 0000000000000000..55b651b455a07ffc --- /dev/null +++ b/test/timeline.cpp @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline test + */ + +#include +#include +#include + +#include + +#include "test.h" +#include "thread.h" +#include "timeline.h" + +using namespace std; +using namespace libcamera; + +enum TimelineTestActionType { + Inc, +}; + +class IncrementAction : public FrameAction +{ +public: + IncrementAction(unsigned int frame, unsigned int *counter) + : FrameAction(Inc, frame), counter_(counter) + { + } + + void run() override + { + (*counter_)++; + } + +private: + unsigned int *counter_; +}; + +class TimelineTest : public Test +{ +public: + void fakeSOE() + { + timeline_.notifyStartOfExposure(sequence_++, + std::chrono::steady_clock::now()); + } + + int init() + { + sequence_ = 0; + return 0; + } + + int run() + { + EventDispatcher *dispatcher = Thread::current()->eventDispatcher(); + unsigned int loops, counter; + Timer timer; + + /* + * Run frames and make sure an event is fired for each and that + * the messured frame interval is true. + */ + loops = 500; + counter = 0; + for (unsigned int i = 0; i < loops; i++) { + timeline_.scheduleAction(new IncrementAction(i + 1, &counter)); + fakeSOE(); + timer.start(10); + while (timer.isRunning()) + dispatcher->processEvents(); + } + + cout << "Got " << counter << " evens, expected " << loops - 1 << endl; + if (counter != loops - 1) + return TestFail; + + unsigned int interval = std::chrono::duration_cast(timeline_.frameInterval()).count(); + cout << "Messured frame interval " << interval << " expected 10" << endl; + if (interval != 10) + return TestFail; + + return TestPass; + } + + void cleanup() + { + } +private: + unsigned int sequence_; + Timeline timeline_; +}; + +TEST_REGISTER(TimelineTest) From patchwork Thu Oct 3 17:49:39 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2091 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id E6EF661767 for ; Thu, 3 Oct 2019 19:50:42 +0200 (CEST) X-Halon-ID: 3408b29e-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 3408b29e-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:53 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:39 +0200 Message-Id: <20191003174941.1296988-10-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 09/11] include: linux: Add rkisp1 kernel header and format definitions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:43 -0000 Add kernel header and format definitions from v8 of the rkisp1 series [1]. The driver is currently out of tree, the header file is not exported as part of the standard kernel uAPI. Headers have been exported on top of a v5.2 kernel tree using scripts/headers_install.sh. 1. https://lore.kernel.org/linux-media/20190730184256.30338-1-helen.koike@collabora.com/ Signed-off-by: Niklas Söderlund Acked-by: Kieran Bingham Acked-by: Laurent Pinchart --- include/linux/rkisp1-config.h | 816 ++++++++++++++++++++++++++++++++++ include/linux/videodev2.h | 4 + 2 files changed, 820 insertions(+) create mode 100644 include/linux/rkisp1-config.h diff --git a/include/linux/rkisp1-config.h b/include/linux/rkisp1-config.h new file mode 100644 index 0000000000000000..f0d1bd82e857fd79 --- /dev/null +++ b/include/linux/rkisp1-config.h @@ -0,0 +1,816 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip isp1 driver + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +/* + * TODO: Improve documentation, mostly regarding abbreviation and hardware + * specificities. + */ + +#ifndef _RKISP1_CONFIG_H +#define _RKISP1_CONFIG_H + +#include +#include + +#define CIFISP_MODULE_DPCC (1 << 0) +#define CIFISP_MODULE_BLS (1 << 1) +#define CIFISP_MODULE_SDG (1 << 2) +#define CIFISP_MODULE_HST (1 << 3) +#define CIFISP_MODULE_LSC (1 << 4) +#define CIFISP_MODULE_AWB_GAIN (1 << 5) +#define CIFISP_MODULE_FLT (1 << 6) +#define CIFISP_MODULE_BDM (1 << 7) +#define CIFISP_MODULE_CTK (1 << 8) +#define CIFISP_MODULE_GOC (1 << 9) +#define CIFISP_MODULE_CPROC (1 << 10) +#define CIFISP_MODULE_AFC (1 << 11) +#define CIFISP_MODULE_AWB (1 << 12) +#define CIFISP_MODULE_IE (1 << 13) +#define CIFISP_MODULE_AEC (1 << 14) +#define CIFISP_MODULE_WDR (1 << 15) +#define CIFISP_MODULE_DPF (1 << 16) +#define CIFISP_MODULE_DPF_STRENGTH (1 << 17) + +#define CIFISP_CTK_COEFF_MAX 0x100 +#define CIFISP_CTK_OFFSET_MAX 0x800 + +#define CIFISP_AE_MEAN_MAX 25 +#define CIFISP_HIST_BIN_N_MAX 16 +#define CIFISP_AFM_MAX_WINDOWS 3 +#define CIFISP_DEGAMMA_CURVE_SIZE 17 + +#define CIFISP_BDM_MAX_TH 0xFF + +/* + * Black level compensation + */ +/* maximum value for horizontal start address */ +#define CIFISP_BLS_START_H_MAX 0x00000FFF +/* maximum value for horizontal stop address */ +#define CIFISP_BLS_STOP_H_MAX 0x00000FFF +/* maximum value for vertical start address */ +#define CIFISP_BLS_START_V_MAX 0x00000FFF +/* maximum value for vertical stop address */ +#define CIFISP_BLS_STOP_V_MAX 0x00000FFF +/* maximum is 2^18 = 262144*/ +#define CIFISP_BLS_SAMPLES_MAX 0x00000012 +/* maximum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MAX 0x00000FFF +/* minimum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MIN 0xFFFFF000 +/* 13 bit range (signed)*/ +#define CIFISP_BLS_FIX_MASK 0x00001FFF + +/* + * Automatic white balance measurments + */ +#define CIFISP_AWB_MAX_GRID 1 +#define CIFISP_AWB_MAX_FRAMES 7 + +/* + * Gamma out + */ +/* Maximum number of color samples supported */ +#define CIFISP_GAMMA_OUT_MAX_SAMPLES 17 + +/* + * Lens shade correction + */ +#define CIFISP_LSC_GRAD_TBL_SIZE 8 +#define CIFISP_LSC_SIZE_TBL_SIZE 8 +/* + * The following matches the tuning process, + * not the max capabilities of the chip. + * Last value unused. + */ +#define CIFISP_LSC_DATA_TBL_SIZE 290 + +/* + * Histogram calculation + */ +/* Last 3 values unused. */ +#define CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE 28 + +/* + * Defect Pixel Cluster Correction + */ +#define CIFISP_DPCC_METHODS_MAX 3 + +/* + * Denoising pre filter + */ +#define CIFISP_DPF_MAX_NLF_COEFFS 17 +#define CIFISP_DPF_MAX_SPATIAL_COEFFS 6 + +/* + * Measurement types + */ +#define CIFISP_STAT_AWB (1 << 0) +#define CIFISP_STAT_AUTOEXP (1 << 1) +#define CIFISP_STAT_AFM_FIN (1 << 2) +#define CIFISP_STAT_HIST (1 << 3) + +enum cifisp_histogram_mode { + CIFISP_HISTOGRAM_MODE_DISABLE, + CIFISP_HISTOGRAM_MODE_RGB_COMBINED, + CIFISP_HISTOGRAM_MODE_R_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_G_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_B_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_Y_HISTOGRAM +}; + +enum cifisp_awb_mode_type { + CIFISP_AWB_MODE_MANUAL, + CIFISP_AWB_MODE_RGB, + CIFISP_AWB_MODE_YCBCR +}; + +enum cifisp_flt_mode { + CIFISP_FLT_STATIC_MODE, + CIFISP_FLT_DYNAMIC_MODE +}; + +/** + * enum cifisp_exp_ctrl_autostop - stop modes + * @CIFISP_EXP_CTRL_AUTOSTOP_0: continuous measurement + * @CIFISP_EXP_CTRL_AUTOSTOP_1: stop measuring after a complete frame + */ +enum cifisp_exp_ctrl_autostop { + CIFISP_EXP_CTRL_AUTOSTOP_0 = 0, + CIFISP_EXP_CTRL_AUTOSTOP_1 = 1, +}; + +/** + * enum cifisp_exp_meas_mode - Exposure measure mode + * @CIFISP_EXP_MEASURING_MODE_0: Y = 16 + 0.25R + 0.5G + 0.1094B + * @CIFISP_EXP_MEASURING_MODE_1: Y = (R + G + B) x (85/256) + */ +enum cifisp_exp_meas_mode { + CIFISP_EXP_MEASURING_MODE_0, + CIFISP_EXP_MEASURING_MODE_1, +}; + +/*---------- PART1: Input Parameters ------------*/ + +struct cifisp_window { + __u16 h_offs; + __u16 v_offs; + __u16 h_size; + __u16 v_size; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_fixed_val - BLS fixed subtraction values + * + * The values will be subtracted from the sensor + * values. Therefore a negative value means addition instead of subtraction! + * + * @r: Fixed (signed!) subtraction value for Bayer pattern R + * @gr: Fixed (signed!) subtraction value for Bayer pattern Gr + * @gb: Fixed (signed!) subtraction value for Bayer pattern Gb + * @b: Fixed (signed!) subtraction value for Bayer pattern B + */ +struct cifisp_bls_fixed_val { + __s16 r; + __s16 gr; + __s16 gb; + __s16 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_config - Configuration used by black level subtraction + * + * @enable_auto: Automatic mode activated means that the measured values + * are subtracted. Otherwise the fixed subtraction + * values will be subtracted. + * @en_windows: enabled window + * @bls_window1: Measurement window 1 size + * @bls_window2: Measurement window 2 size + * @bls_samples: Set amount of measured pixels for each Bayer position + * (A, B,C and D) to 2^bls_samples. + * @cifisp_bls_fixed_val: Fixed subtraction values + */ +struct cifisp_bls_config { + __u8 enable_auto; + __u8 en_windows; + struct cifisp_window bls_window1; + struct cifisp_window bls_window2; + __u8 bls_samples; + struct cifisp_bls_fixed_val fixed_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Methods Configuration used by DPCC + * + * Methods Configuration used by Defect Pixel Cluster Correction + * + * @method: Method enable bits + * @line_thresh: Line threshold + * @line_mad_fac: Line MAD factor + * @pg_fac: Peak gradient factor + * @rnd_thresh: Rank Neighbor Difference threshold + * @rg_fac: Rank gradient factor + */ +struct cifisp_dpcc_methods_config { + __u32 method; + __u32 line_thresh; + __u32 line_mad_fac; + __u32 pg_fac; + __u32 rnd_thresh; + __u32 rg_fac; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Configuration used by DPCC + * + * Configuration used by Defect Pixel Cluster Correction + * + * @mode: dpcc output mode + * @output_mode: whether use hard coded methods + * @set_use: stage1 methods set + * @methods: methods config + * @ro_limits: rank order limits + * @rnd_offs: differential rank offsets for rank neighbor difference + */ +struct cifisp_dpcc_config { + __u32 mode; + __u32 output_mode; + __u32 set_use; + struct cifisp_dpcc_methods_config methods[CIFISP_DPCC_METHODS_MAX]; + __u32 ro_limits; + __u32 rnd_offs; +} __attribute__ ((packed)); + +struct cifisp_gamma_corr_curve { + __u16 gamma_y[CIFISP_DEGAMMA_CURVE_SIZE]; +} __attribute__ ((packed)); + +struct cifisp_gamma_curve_x_axis_pnts { + __u32 gamma_dx0; + __u32 gamma_dx1; +} __attribute__ ((packed)); + +/** + * struct cifisp_gamma_corr_curve - Configuration used by sensor degamma + * + * @curve_x: gamma curve point definition axis for x + * @xa_pnts: x increments + */ +struct cifisp_sdg_config { + struct cifisp_gamma_corr_curve curve_r; + struct cifisp_gamma_corr_curve curve_g; + struct cifisp_gamma_corr_curve curve_b; + struct cifisp_gamma_curve_x_axis_pnts xa_pnts; +} __attribute__ ((packed)); + +/** + * struct cifisp_lsc_config - Configuration used by Lens shading correction + * + * refer to REF_01 for details + */ +struct cifisp_lsc_config { + __u32 r_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gr_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gb_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 b_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + + __u32 x_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + __u32 y_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + + __u32 x_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u32 y_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u16 config_width; + __u16 config_height; +} __attribute__ ((packed)); + +/** + * struct cifisp_ie_config - Configuration used by image effects + * + * @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1 + * @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2 + * @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1 + * @eff_mat_4: 3x3 Matrix Coefficients for Sketch Effect 2 + * @eff_mat_5: 3x3 Matrix Coefficients for Sketch Effect 3 + * @eff_tint: Chrominance increment values of tint (used for sepia effect) + */ +struct cifisp_ie_config { + __u16 effect; + __u16 color_sel; + __u16 eff_mat_1; + __u16 eff_mat_2; + __u16 eff_mat_3; + __u16 eff_mat_4; + __u16 eff_mat_5; + __u16 eff_tint; +} __attribute__ ((packed)); + +/** + * struct cifisp_cproc_config - Configuration used by Color Processing + * + * @c_out_range: Chrominance pixel clipping range at output. + * (0 for limit, 1 for full) + * @y_in_range: Luminance pixel clipping range at output. + * @y_out_range: Luminance pixel clipping range at output. + * @contrast: 00~ff, 0.0~1.992 + * @brightness: 80~7F, -128~+127 + * @sat: saturation, 00~FF, 0.0~1.992 + * @hue: 80~7F, -90~+87.188 + */ +struct cifisp_cproc_config { + __u8 c_out_range; + __u8 y_in_range; + __u8 y_out_range; + __u8 contrast; + __u8 brightness; + __u8 sat; + __u8 hue; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_meas_config - Configuration used by auto white balance + * + * @awb_wnd: white balance measurement window (in pixels) + * (from enum cifisp_awb_mode_type) + * @max_y: only pixels values < max_y contribute to awb measurement, set to 0 + * to disable this feature + * @min_y: only pixels values > min_y contribute to awb measurement + * @max_csum: Chrominance sum maximum value, only consider pixels with Cb+Cr, + * smaller than threshold for awb measurements + * @min_c: Chrominance minimum value, only consider pixels with Cb/Cr + * each greater than threshold value for awb measurements + * @frames: number of frames - 1 used for mean value calculation + * (ucFrames=0 means 1 Frame) + * @awb_ref_cr: reference Cr value for AWB regulation, target for AWB + * @awb_ref_cb: reference Cb value for AWB regulation, target for AWB + */ +struct cifisp_awb_meas_config { + /* + * Note: currently the h and v offsets are mapped to grid offsets + */ + struct cifisp_window awb_wnd; + __u32 awb_mode; + __u8 max_y; + __u8 min_y; + __u8 max_csum; + __u8 min_c; + __u8 frames; + __u8 awb_ref_cr; + __u8 awb_ref_cb; + __u8 enable_ymax_cmp; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_gain_config - Configuration used by auto white balance gain + * + * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8 + */ +struct cifisp_awb_gain_config { + __u16 gain_red; + __u16 gain_green_r; + __u16 gain_blue; + __u16 gain_green_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_flt_config - Configuration used by ISP filtering + * + * @mode: ISP_FILT_MODE register fields (from enum cifisp_flt_mode) + * @grn_stage1: ISP_FILT_MODE register fields + * @chr_h_mode: ISP_FILT_MODE register fields + * @chr_v_mode: ISP_FILT_MODE register fields + * + * refer to REF_01 for details. + */ + +struct cifisp_flt_config { + __u32 mode; + __u8 grn_stage1; + __u8 chr_h_mode; + __u8 chr_v_mode; + __u32 thresh_bl0; + __u32 thresh_bl1; + __u32 thresh_sh0; + __u32 thresh_sh1; + __u32 lum_weight; + __u32 fac_sh1; + __u32 fac_sh0; + __u32 fac_mid; + __u32 fac_bl0; + __u32 fac_bl1; +} __attribute__ ((packed)); + +/** + * struct cifisp_bdm_config - Configuration used by Bayer DeMosaic + * + * @demosaic_th: threshod for bayer demosaicing texture detection + */ +struct cifisp_bdm_config { + __u8 demosaic_th; +} __attribute__ ((packed)); + +/** + * struct cifisp_ctk_config - Configuration used by Cross Talk correction + * + * @coeff: color correction matrix + * @ct_offset_b: offset for the crosstalk correction matrix + */ +struct cifisp_ctk_config { + __u16 coeff0; + __u16 coeff1; + __u16 coeff2; + __u16 coeff3; + __u16 coeff4; + __u16 coeff5; + __u16 coeff6; + __u16 coeff7; + __u16 coeff8; + __u16 ct_offset_r; + __u16 ct_offset_g; + __u16 ct_offset_b; +} __attribute__ ((packed)); + +enum cifisp_goc_mode { + CIFISP_GOC_MODE_LOGARITHMIC, + CIFISP_GOC_MODE_EQUIDISTANT +}; + +/** + * struct cifisp_goc_config - Configuration used by Gamma Out correction + * + * @mode: goc mode (from enum cifisp_goc_mode) + * @gamma_y: gamma out curve y-axis for all color components + */ +struct cifisp_goc_config { + __u32 mode; + __u16 gamma_y[CIFISP_GAMMA_OUT_MAX_SAMPLES]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hst_config - Configuration used by Histogram + * + * @mode: histogram mode (from enum cifisp_histogram_mode) + * @histogram_predivider: process every stepsize pixel, all other pixels are + * skipped + * @meas_window: coordinates of the measure window + * @hist_weight: weighting factor for sub-windows + */ +struct cifisp_hst_config { + __u32 mode; + __u8 histogram_predivider; + struct cifisp_window meas_window; + __u8 hist_weight[CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE]; +} __attribute__ ((packed)); + +/** + * struct cifisp_aec_config - Configuration used by Auto Exposure Control + * + * @mode: Exposure measure mode (from enum cifisp_exp_meas_mode) + * @autostop: stop mode (from enum cifisp_exp_ctrl_autostop) + * @meas_window: coordinates of the measure window + */ +struct cifisp_aec_config { + __u32 mode; + __u32 autostop; + struct cifisp_window meas_window; +} __attribute__ ((packed)); + +/** + * struct cifisp_afc_config - Configuration used by Auto Focus Control + * + * @num_afm_win: max CIFISP_AFM_MAX_WINDOWS + * @afm_win: coordinates of the meas window + * @thres: threshold used for minimizing the influence of noise + * @var_shift: the number of bits for the shift operation at the end of the + * calculation chain. + */ +struct cifisp_afc_config { + __u8 num_afm_win; + struct cifisp_window afm_win[CIFISP_AFM_MAX_WINDOWS]; + __u32 thres; + __u32 var_shift; +} __attribute__ ((packed)); + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_GAIN_USAGE_DISABLED: don't use any gains in preprocessing stage + * @CIFISP_DPF_GAIN_USAGE_NF_GAINS: use only the noise function gains from + * registers DPF_NF_GAIN_R, ... + * @CIFISP_DPF_GAIN_USAGE_LSC_GAINS: use only the gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS: use the noise function gains and the + * gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_AWB_GAINS: use only the gains from AWB module + * @CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: use the gains from AWB and LSC module + * @CIFISP_DPF_GAIN_USAGE_MAX: upper border (only for an internal evaluation) + */ +enum cifisp_dpf_gain_usage { + CIFISP_DPF_GAIN_USAGE_DISABLED, + CIFISP_DPF_GAIN_USAGE_NF_GAINS, + CIFISP_DPF_GAIN_USAGE_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_MAX +}; + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_RB_FILTERSIZE_13x9: red and blue filter kernel size 13x9 + * (means 7x5 active pixel) + * @CIFISP_DPF_RB_FILTERSIZE_9x9: red and blue filter kernel size 9x9 + * (means 5x5 active pixel) + */ +enum cifisp_dpf_rb_filtersize { + CIFISP_DPF_RB_FILTERSIZE_13x9, + CIFISP_DPF_RB_FILTERSIZE_9x9, +}; + +/** + * enum cifisp_dpf_nll_scale_mode - dpf noise level scale mode + * @CIFISP_NLL_SCALE_LINEAR: use a linear scaling + * @CIFISP_NLL_SCALE_LOGARITHMIC: use a logarithmic scaling + */ +enum cifisp_dpf_nll_scale_mode { + CIFISP_NLL_SCALE_LINEAR, + CIFISP_NLL_SCALE_LOGARITHMIC, +}; + +/** + * struct cifisp_dpf_nll - Noise level lookup + * + * @coeff: Noise level Lookup coefficient + * @scale_mode: dpf noise level scale mode (from enum cifisp_dpf_nll_scale_mode) + */ +struct cifisp_dpf_nll { + __u16 coeff[CIFISP_DPF_MAX_NLF_COEFFS]; + __u32 scale_mode; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_rb_flt - Red blue filter config + * + * @fltsize: The filter size for the red and blue pixels + * (from enum cifisp_dpf_rb_filtersize) + * @spatial_coeff: Spatial weights + * @r_enable: enable filter processing for red pixels + * @b_enable: enable filter processing for blue pixels + */ +struct cifisp_dpf_rb_flt { + __u32 fltsize; + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 r_enable; + __u8 b_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_g_flt - Green filter Configuration + * + * @spatial_coeff: Spatial weights + * @gr_enable: enable filter processing for green pixels in green/red lines + * @gb_enable: enable filter processing for green pixels in green/blue lines + */ +struct cifisp_dpf_g_flt { + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 gr_enable; + __u8 gb_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_gain - Noise function Configuration + * + * @mode: dpf gain usage (from enum cifisp_dpf_gain_usage) + * @nf_r_gain: Noise function Gain that replaces the AWB gain for red pixels + * @nf_b_gain: Noise function Gain that replaces the AWB gain for blue pixels + * @nf_gr_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a red line + * @nf_gb_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a blue line + */ +struct cifisp_dpf_gain { + __u32 mode; + __u16 nf_r_gain; + __u16 nf_b_gain; + __u16 nf_gr_gain; + __u16 nf_gb_gain; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_config - Configuration used by De-noising pre-filter + * + * @gain: noise function gain + * @g_flt: green filter config + * @rb_flt: red blue filter config + * @nll: noise level lookup + */ +struct cifisp_dpf_config { + struct cifisp_dpf_gain gain; + struct cifisp_dpf_g_flt g_flt; + struct cifisp_dpf_rb_flt rb_flt; + struct cifisp_dpf_nll nll; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_strength_config - strength of the filter + * + * @r: filter strength of the RED filter + * @g: filter strength of the GREEN filter + * @b: filter strength of the BLUE filter + */ +struct cifisp_dpf_strength_config { + __u8 r; + __u8 g; + __u8 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_other_cfg - Parameters for some blocks in rockchip isp1 + * + * @dpcc_config: Defect Pixel Cluster Correction config + * @bls_config: Black Level Subtraction config + * @sdg_config: sensor degamma config + * @lsc_config: Lens Shade config + * @awb_gain_config: Auto White balance gain config + * @flt_config: filter config + * @bdm_config: demosaic config + * @ctk_config: cross talk config + * @goc_config: gamma out config + * @bls_config: black level subtraction config + * @dpf_config: De-noising pre-filter config + * @dpf_strength_config: dpf strength config + * @cproc_config: color process config + * @ie_config: image effects config + */ +struct cifisp_isp_other_cfg { + struct cifisp_dpcc_config dpcc_config; + struct cifisp_bls_config bls_config; + struct cifisp_sdg_config sdg_config; + struct cifisp_lsc_config lsc_config; + struct cifisp_awb_gain_config awb_gain_config; + struct cifisp_flt_config flt_config; + struct cifisp_bdm_config bdm_config; + struct cifisp_ctk_config ctk_config; + struct cifisp_goc_config goc_config; + struct cifisp_dpf_config dpf_config; + struct cifisp_dpf_strength_config dpf_strength_config; + struct cifisp_cproc_config cproc_config; + struct cifisp_ie_config ie_config; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_meas_cfg - Rockchip ISP1 Measure Parameters + * + * @awb_meas_config: auto white balance config + * @hst_config: histogram config + * @aec_config: auto exposure config + * @afc_config: auto focus config + */ +struct cifisp_isp_meas_cfg { + struct cifisp_awb_meas_config awb_meas_config; + struct cifisp_hst_config hst_config; + struct cifisp_aec_config aec_config; + struct cifisp_afc_config afc_config; +} __attribute__ ((packed)); + +/** + * struct rkisp1_isp_params_cfg - Rockchip ISP1 Input Parameters Meta Data + * + * @module_en_update: mask the enable bits of which module should be updated + * @module_ens: mask the enable value of each module, only update the module + * which correspond bit was set in module_en_update + * @module_cfg_update: mask the config bits of which module should be updated + * @meas: measurement config + * @others: other config + */ +struct rkisp1_isp_params_cfg { + __u32 module_en_update; + __u32 module_ens; + __u32 module_cfg_update; + + struct cifisp_isp_meas_cfg meas; + struct cifisp_isp_other_cfg others; +} __attribute__ ((packed)); + +/*---------- PART2: Measurement Statistics ------------*/ + +/** + * struct cifisp_bls_meas_val - AWB measured values + * + * @cnt: White pixel count, number of "white pixels" found during laster + * measurement + * @mean_y_or_g: Mean value of Y within window and frames, + * Green if RGB is selected. + * @mean_cb_or_b: Mean value of Cb within window and frames, + * Blue if RGB is selected. + * @mean_cr_or_r: Mean value of Cr within window and frames, + * Red if RGB is selected. + */ +struct cifisp_awb_meas { + __u32 cnt; + __u8 mean_y_or_g; + __u8 mean_cb_or_b; + __u8 mean_cr_or_r; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_stat - statistics automatic white balance data + * + * @awb_mean: Mean measured data + */ +struct cifisp_awb_stat { + struct cifisp_awb_meas awb_mean[CIFISP_AWB_MAX_GRID]; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_meas_val - BLS measured values + * + * @meas_r: Mean measured value for Bayer pattern R + * @meas_gr: Mean measured value for Bayer pattern Gr + * @meas_gb: Mean measured value for Bayer pattern Gb + * @meas_b: Mean measured value for Bayer pattern B + */ +struct cifisp_bls_meas_val { + __u16 meas_r; + __u16 meas_gr; + __u16 meas_gb; + __u16 meas_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_ae_stat - statistics auto exposure data + * + * @exp_mean: Mean luminance value of block xx + * @bls_val: BLS measured values + * + * Image is divided into 5x5 blocks. + */ +struct cifisp_ae_stat { + __u8 exp_mean[CIFISP_AE_MEAN_MAX]; + struct cifisp_bls_meas_val bls_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_meas_val - AF measured values + * + * @sum: sharpness, refer to REF_01 for definition + * @lum: luminance, refer to REF_01 for definition + */ +struct cifisp_af_meas_val { + __u32 sum; + __u32 lum; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_stat - statistics auto focus data + * + * @window: AF measured value of window x + * + * The module measures the sharpness in 3 windows of selectable size via + * register settings(ISP_AFM_*_A/B/C) + */ +struct cifisp_af_stat { + struct cifisp_af_meas_val window[CIFISP_AFM_MAX_WINDOWS]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hist_stat - statistics histogram data + * + * @hist_bins: measured bin counters + * + * Measurement window divided into 25 sub-windows, set + * with ISP_HIST_XXX + */ +struct cifisp_hist_stat { + __u16 hist_bins[CIFISP_HIST_BIN_N_MAX]; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Data + * + * @cifisp_awb_stat: statistics data for automatic white balance + * @cifisp_ae_stat: statistics data for auto exposure + * @cifisp_af_stat: statistics data for auto focus + * @cifisp_hist_stat: statistics histogram data + */ +struct cifisp_stat { + struct cifisp_awb_stat awb; + struct cifisp_ae_stat ae; + struct cifisp_af_stat af; + struct cifisp_hist_stat hist; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Meta Data + * + * @meas_type: measurement types (CIFISP_STAT_ definitions) + * @frame_id: frame ID for sync + * @params: statistics data + */ +struct rkisp1_stat_buffer { + __u32 meas_type; + __u32 frame_id; + struct cifisp_stat params; +} __attribute__ ((packed)); + +#endif /* _RKISP1_CONFIG_H */ diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index 5e739270116d3a21..94304bfeb4b6b9ed 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -727,6 +727,10 @@ struct v4l2_pix_format { #define V4L2_META_FMT_UVC v4l2_fourcc('U', 'V', 'C', 'H') /* UVC Payload Header metadata */ #define V4L2_META_FMT_D4XX v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */ +/* Vendor specific - used for RK_ISP1 camera sub-system */ +#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P') /* Rockchip ISP1 params */ +#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A statistics */ + /* priv field value to indicates that subsequent fields are valid. */ #define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe From patchwork Thu Oct 3 17:49:40 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2092 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D091361767 for ; Thu, 3 Oct 2019 19:50:43 +0200 (CEST) X-Halon-ID: 34f93834-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 34f93834-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:54 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:40 +0200 Message-Id: <20191003174941.1296988-11-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 10/11] libcamera: ipa: rkisp1: Add basic control of auto exposure X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:44 -0000 Add an IPA which controls the exposure time and analog gain for a sensor connected to the rkisp1 pipeline. The IPA supports turning AE on and off and informing the camera of the status of the AE control loop. Signed-off-by: Niklas Söderlund --- include/ipa/rkisp1.h | 23 ++++ src/ipa/ipa_rkisp1.cpp | 278 +++++++++++++++++++++++++++++++++++++++++ src/ipa/meson.build | 13 ++ 3 files changed, 314 insertions(+) create mode 100644 include/ipa/rkisp1.h create mode 100644 src/ipa/ipa_rkisp1.cpp diff --git a/include/ipa/rkisp1.h b/include/ipa/rkisp1.h new file mode 100644 index 0000000000000000..de9c7971e7bc94ad --- /dev/null +++ b/include/ipa/rkisp1.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Image Processing Algorithm interface for RkISP1 + */ +#ifndef __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ +#define __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ + +enum RkISP1BufferType { + RKISP1_BUFFER_PARAM = 1, + RKISP1_BUFFER_STAT = 2, +}; + +enum RkISP1Operations { + RKISP1_IPA_ACTION_V4L2_SET = 1, + RKISP1_IPA_ACTION_QUEUE_BUFFER = 2, + RKISP1_IPA_ACTION_META_DATA = 3, + RKISP1_IPA_EVENT_SIGNAL_BUFFER = 4, + RKISP1_IPA_EVENT_QUEUE_REQUEST = 5, +}; + +#endif /* __LIBCAMERA_IPA_INTERFACE_RKISP1_H__ */ diff --git a/src/ipa/ipa_rkisp1.cpp b/src/ipa/ipa_rkisp1.cpp new file mode 100644 index 0000000000000000..78dd148a66299ab9 --- /dev/null +++ b/src/ipa/ipa_rkisp1.cpp @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * ipa_rkisp1.cpp - RkISP1 Image Processing Algorithms + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "utils.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPARkISP1) + +class IPARkISP1 : public IPAInterface +{ +public: + int init() override { return 0; } + + void configure(const std::map &streamConfig, + const std::map &entityControls) override; + void mapBuffers(const std::vector &buffers) override; + void unmapBuffers(const std::vector &buffers) override; + void processEvent(const IPAOperationData &event) override; + +private: + void signalBuffer(unsigned int type, unsigned int id); + void queueRequest(unsigned int frame, const ControlList &controls); + void updateStatistics(unsigned int frame, BufferMemory &statistics); + + void setControls(unsigned int frame); + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metaDataReady(unsigned int frame, unsigned int aeState); + + std::map> bufferInfo_; + std::map> bufferFrame_; + std::map> bufferFree_; + + /* Camera sensor controls. */ + bool autoExposure_; + uint64_t exposure_; + uint64_t minExposure_; + uint64_t maxExposure_; + uint64_t gain_; + uint64_t minGain_; + uint64_t maxGain_; +}; + +void IPARkISP1::configure(const std::map &streamConfig, + const std::map &entityControls) +{ + if (entityControls.empty()) + return; + + const V4L2ControlInfoMap &ctrls = entityControls.at(0); + + const auto itExp = ctrls.find(V4L2_CID_EXPOSURE); + if (itExp == ctrls.end()) { + LOG(IPARkISP1, Error) << "Can't find exposure control"; + return; + } + + const auto itGain = ctrls.find(V4L2_CID_ANALOGUE_GAIN); + if (itGain == ctrls.end()) { + LOG(IPARkISP1, Error) << "Can't find gain control"; + return; + } + + autoExposure_ = true; + + minExposure_ = std::max(itExp->second.range().min().get(), 1); + maxExposure_ = itExp->second.range().max().get(); + exposure_ = minExposure_; + + minGain_ = std::max(itGain->second.range().min().get(), 1); + maxGain_ = itGain->second.range().max().get(); + gain_ = minGain_; + + LOG(IPARkISP1, Info) + << "Exposure: " << minExposure_ << "-" << maxExposure_ + << " Gain: " << minGain_ << "-" << maxGain_; + + setControls(0); +} + +void IPARkISP1::mapBuffers(const std::vector &buffers) +{ + for (IPABuffer buffer : buffers) { + bufferInfo_[buffer.type][buffer.id] = buffer.buffer; + bufferInfo_[buffer.type][buffer.id].planes()[0].mem(); + bufferFree_[buffer.type].push(buffer.id); + } +} + +void IPARkISP1::unmapBuffers(const std::vector &buffers) +{ + for (IPABuffer buffer : buffers) + bufferInfo_[buffer.type].erase(buffer.id); +} + +void IPARkISP1::processEvent(const IPAOperationData &event) +{ + switch (event.operation) { + case RKISP1_IPA_EVENT_SIGNAL_BUFFER: + signalBuffer(event.data[0], event.data[1]); + break; + case RKISP1_IPA_EVENT_QUEUE_REQUEST: + queueRequest(event.data[0], event.controls[0]); + break; + default: + LOG(IPARkISP1, Error) << "Unkown event " << event.operation; + break; + } +} + +void IPARkISP1::signalBuffer(unsigned int type, unsigned int id) +{ + if (type == RKISP1_BUFFER_STAT) { + unsigned int frame = bufferFrame_[type][id]; + BufferMemory &mem = bufferInfo_[type][id]; + updateStatistics(frame, mem); + } + + bufferFree_[type].push(id); +} + +void IPARkISP1::queueRequest(unsigned int frame, const ControlList &controls) +{ + /* Find buffers. */ + if (bufferFree_[RKISP1_BUFFER_PARAM].empty()) { + LOG(IPARkISP1, Error) << "Param buffer underrun"; + return; + } + + if (bufferFree_[RKISP1_BUFFER_STAT].empty()) { + LOG(IPARkISP1, Error) << "Statistics buffer underrun"; + return; + } + + unsigned int paramid = bufferFree_[RKISP1_BUFFER_PARAM].front(); + bufferFree_[RKISP1_BUFFER_PARAM].pop(); + unsigned int statid = bufferFree_[RKISP1_BUFFER_STAT].front(); + bufferFree_[RKISP1_BUFFER_STAT].pop(); + + bufferFrame_[RKISP1_BUFFER_PARAM][paramid] = frame; + bufferFrame_[RKISP1_BUFFER_STAT][statid] = frame; + + /* Prepare parameters buffer. */ + BufferMemory &mem = bufferInfo_[RKISP1_BUFFER_PARAM][paramid]; + rkisp1_isp_params_cfg *params = + static_cast(mem.planes()[0].mem()); + + memset(params, 0, sizeof(*params)); + + /* Auto Exposure on/off. */ + if (controls.contains(controls::AeEnable)) { + autoExposure_ = controls.get(controls::AeEnable); + if (autoExposure_) + params->module_ens = CIFISP_MODULE_AEC; + + params->module_en_update = CIFISP_MODULE_AEC; + } + + /* Queue buffers to pipeline. */ + queueBuffer(frame, RKISP1_BUFFER_PARAM, paramid); + queueBuffer(frame, RKISP1_BUFFER_STAT, statid); +} + +void IPARkISP1::updateStatistics(unsigned int frame, BufferMemory &statistics) +{ + const rkisp1_stat_buffer *stats = + static_cast(statistics.planes()[0].mem()); + const cifisp_stat *params = &stats->params; + unsigned int aeState = 0; + + if (stats->meas_type & CIFISP_STAT_AUTOEXP) { + const cifisp_ae_stat *ae = ¶ms->ae; + + const unsigned int target = 60; + + unsigned int value = 0; + unsigned int num = 0; + for (int i = 0; i < CIFISP_AE_MEAN_MAX; i++) { + if (ae->exp_mean[i] > 15) { + value += ae->exp_mean[i]; + num++; + } + } + value /= num; + + double factor = (double)target / value; + + if (frame % 3 == 0) { + double tmp; + + tmp = factor * exposure_ * gain_ / minGain_; + exposure_ = utils::clamp((uint64_t)tmp, minExposure_, maxExposure_); + + tmp = tmp / exposure_ * minGain_; + gain_ = utils::clamp((uint64_t)tmp, minGain_, maxGain_); + + setControls(frame + 1); + } + + aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1; + } + + metaDataReady(frame, aeState); +} + +void IPARkISP1::setControls(unsigned int frame) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_V4L2_SET; + op.data.push_back(frame); + op.data.push_back(V4L2_CID_EXPOSURE); + op.data.push_back(exposure_); + op.data.push_back(V4L2_CID_ANALOGUE_GAIN); + op.data.push_back(gain_); + + queueFrameAction.emit(op); +} + +void IPARkISP1::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_QUEUE_BUFFER; + op.data = { frame, type, id }; + + queueFrameAction.emit(op); +} + +void IPARkISP1::metaDataReady(unsigned int frame, unsigned int aeState) +{ + IPAOperationData op; + op.operation = RKISP1_IPA_ACTION_META_DATA; + op.data = { frame, aeState }; + + queueFrameAction.emit(op); +} + +/* + * External IPA module interface + */ + +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 1, + "PipelineHandlerRkISP1", + "RkISP1 IPA", + "LGPL-2.1-or-later", +}; + +IPAInterface *ipaCreate() +{ + return new IPARkISP1(); +} +}; + +}; /* namespace libcamera */ diff --git a/src/ipa/meson.build b/src/ipa/meson.build index 10448c2ffc76af4b..eb5b852eec282735 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -3,6 +3,10 @@ ipa_dummy_sources = [ ['ipa_dummy_isolate', 'Proprietary'], ] +ipa_sources = [ + ['ipa_rkisp1', 'ipa_rkisp1.cpp', 'LGPL-2.1-or-later'], +] + ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources @@ -14,5 +18,14 @@ foreach t : ipa_dummy_sources cpp_args : '-DLICENSE="' + t[1] + '"') endforeach +foreach t : ipa_sources + ipa = shared_module(t[0], t[1], + name_prefix : '', + include_directories : includes, + install : true, + install_dir : ipa_install_dir, + cpp_args : '-DLICENSE="' + t[2] + '"') +endforeach + config_h.set('IPA_MODULE_DIR', '"' + join_paths(get_option('prefix'), ipa_install_dir) + '"') From patchwork Thu Oct 3 17:49:41 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2093 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6069660BE9 for ; Thu, 3 Oct 2019 19:50:45 +0200 (CEST) X-Halon-ID: 3573d54b-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 3573d54b-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:55 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:41 +0200 Message-Id: <20191003174941.1296988-12-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 11/11] libcamera: pipeline: rkisp1: Attach to an IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:45 -0000 Add the plumbing to the pipeline handler to interact with an IPA module. This change makes the usage of an IPA module mandatory for the rkisp1 pipeline. Signed-off-by: Niklas Söderlund --- src/libcamera/pipeline/rkisp1/meson.build | 1 + src/libcamera/pipeline/rkisp1/rkisp1.cpp | 326 ++++++++++++++++++++- src/libcamera/pipeline/rkisp1/rkisp1.h | 78 +++++ src/libcamera/pipeline/rkisp1/timeline.cpp | 56 ++++ 4 files changed, 448 insertions(+), 13 deletions(-) create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1.h create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build index f1cc4046b5d064cb..d04fb45223e72fa1 100644 --- a/src/libcamera/pipeline/rkisp1/meson.build +++ b/src/libcamera/pipeline/rkisp1/meson.build @@ -1,3 +1,4 @@ libcamera_sources += files([ 'rkisp1.cpp', + 'timeline.cpp', ]) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index de4ab523d0e4fe36..ade8c8823a0ad99d 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -5,20 +5,22 @@ * rkisp1.cpp - Pipeline handler for Rockchip ISP1 */ +#include "rkisp1.h" + #include #include #include #include -#include #include #include +#include #include #include -#include "camera_sensor.h" #include "device_enumerator.h" +#include "ipa_manager.h" #include "log.h" #include "media_device.h" #include "pipeline_handler.h" @@ -34,7 +36,7 @@ class RkISP1CameraData : public CameraData { public: RkISP1CameraData(PipelineHandler *pipe) - : CameraData(pipe), sensor_(nullptr) + : CameraData(pipe), sensor_(nullptr), frame_(0) { } @@ -43,8 +45,21 @@ public: delete sensor_; } + int loadIPA(); + Stream stream_; CameraSensor *sensor_; + unsigned int frame_; + std::vector ipaBuffers_; + std::map frameInfo_; + RkISP1Timeline timeline_; + +private: + void queueFrameAction(const IPAOperationData &action); + + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metaDataReady(unsigned int frame, unsigned int aeState); }; class RkISP1CameraConfiguration : public CameraConfiguration @@ -99,18 +114,114 @@ private: PipelineHandler::cameraData(camera)); } + friend RkISP1CameraData; + int initLinks(); int createCamera(MediaEntity *sensor); + void tryCompleteRequest(Request *request); void bufferReady(Buffer *buffer); + void paramReady(Buffer *buffer); + void statReady(Buffer *buffer); MediaDevice *media_; V4L2Subdevice *dphy_; V4L2Subdevice *isp_; V4L2VideoDevice *video_; + V4L2VideoDevice *param_; + V4L2VideoDevice *stat_; + + BufferPool paramPool_; + BufferPool statPool_; + + std::map paramBuffers_; + std::map statBuffers_; Camera *activeCamera_; }; +int RkISP1CameraData::loadIPA() +{ + ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->queueFrameAction.connect(this, + &RkISP1CameraData::queueFrameAction); + + return 0; +} + +void RkISP1CameraData::queueFrameAction(const IPAOperationData &action) +{ + switch (action.operation) { + case RKISP1_IPA_ACTION_V4L2_SET: { + unsigned int frame = action.data[0]; + V4L2ControlList controls; + for (unsigned int i = 1; i < action.data.size(); i += 2) + controls.add(action.data[i], action.data[i + 1]); + timeline_.scheduleAction(new RkISP1ActionSetSensor(frame, sensor_, controls)); + break; + } + case RKISP1_IPA_ACTION_QUEUE_BUFFER: { + unsigned int frame = action.data[0]; + unsigned int type = action.data[1]; + unsigned int id = action.data[2]; + queueBuffer(frame, type, id); + break; + } + case RKISP1_IPA_ACTION_META_DATA: { + unsigned int frame = action.data[0]; + unsigned aeState = action.data[1]; + metaDataReady(frame, aeState); + break; + } + default: + LOG(RkISP1, Error) << "Unkown action " << action.operation; + break; + } +} + +void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + RkISP1ActionType acttype; + V4L2VideoDevice *device; + Buffer *buffer; + switch (type) { + case RKISP1_BUFFER_PARAM: + acttype = QueueParameters; + device = pipe->param_; + buffer = pipe->paramBuffers_[id]; + break; + case RKISP1_BUFFER_STAT: + acttype = QueueStatistics; + device = pipe->stat_; + buffer = pipe->statBuffers_[id]; + break; + default: + LOG(RkISP1, Error) << "Unkown IPA buffer type " << type; + return; + } + + timeline_.scheduleAction(new RkISP1ActionQueueBuffer(frame, acttype, + device, buffer)); +} + +void RkISP1CameraData::metaDataReady(unsigned int frame, unsigned int aeStatus) +{ + Request *request = frameInfo_[frame]; + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + if (aeStatus) + request->metaData().set(controls::AeLocked, aeStatus == 2); + + pipe->tryCompleteRequest(request); +} + RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera, RkISP1CameraData *data) : CameraConfiguration() @@ -202,12 +313,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) : PipelineHandler(manager), dphy_(nullptr), isp_(nullptr), - video_(nullptr) + video_(nullptr), param_(nullptr), stat_(nullptr) { } PipelineHandlerRkISP1::~PipelineHandlerRkISP1() { + delete param_; + delete stat_; delete video_; delete isp_; delete dphy_; @@ -317,6 +430,20 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) if (ret) return ret; + V4L2DeviceFormat paramFormat = {}; + paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS; + + ret = param_->setFormat(¶mFormat); + if (ret) + return ret; + + V4L2DeviceFormat statFormat = {}; + statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A; + + ret = stat_->setFormat(&statFormat); + if (ret) + return ret; + if (outputFormat.size != cfg.size || outputFormat.fourcc != cfg.pixelFormat) { LOG(RkISP1, Error) @@ -332,39 +459,133 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) int PipelineHandlerRkISP1::allocateBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); Stream *stream = *streams.begin(); + int ret; if (stream->memoryType() == InternalMemory) - return video_->exportBuffers(&stream->bufferPool()); + ret = video_->exportBuffers(&stream->bufferPool()); else - return video_->importBuffers(&stream->bufferPool()); + ret = video_->importBuffers(&stream->bufferPool()); + + if (ret) + return ret; + + paramPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = param_->exportBuffers(¶mPool_); + if (ret) { + video_->releaseBuffers(); + return ret; + } + + statPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = stat_->exportBuffers(&statPool_); + if (ret) { + param_->releaseBuffers(); + video_->releaseBuffers(); + return ret; + } + + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) { + paramBuffers_[i] = new Buffer(i); + statBuffers_[i] = new Buffer(i); + + data->ipaBuffers_.push_back({ + .id = i, + .type = RKISP1_BUFFER_PARAM, + .buffer = paramPool_.buffers()[i], + }); + data->ipaBuffers_.push_back({ + .id = i, + .type = RKISP1_BUFFER_STAT, + .buffer = statPool_.buffers()[i], + }); + } + + data->ipa_->mapBuffers(data->ipaBuffers_); + + return ret; } int PipelineHandlerRkISP1::freeBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); + + data->ipa_->unmapBuffers(data->ipaBuffers_); + data->ipaBuffers_.clear(); + + for (auto it : paramBuffers_) + delete it.second; + + paramBuffers_.clear(); + + for (auto it : statBuffers_) + delete it.second; + + statBuffers_.clear(); + + if (param_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release parameters buffers"; + + if (stat_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release stat buffers"; + if (video_->releaseBuffers()) - LOG(RkISP1, Error) << "Failed to release buffers"; + LOG(RkISP1, Error) << "Failed to release video buffers"; return 0; } int PipelineHandlerRkISP1::start(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; + ret = param_->streamOn(); + if (ret) { + LOG(RkISP1, Error) + << "Failed to start parameters " << camera->name(); + return ret; + } + + ret = stat_->streamOn(); + if (ret) { + param_->streamOff(); + LOG(RkISP1, Error) + << "Failed to start statistics " << camera->name(); + return ret; + } + ret = video_->streamOn(); - if (ret) + if (ret) { + param_->streamOff(); + stat_->streamOff(); + LOG(RkISP1, Error) << "Failed to start camera " << camera->name(); + } activeCamera_ = camera; + /* Inform IPA of stream configuration and sensor controls. */ + std::map streamConfig; + streamConfig[0] = { + .pixelFormat = data->stream_.configuration().pixelFormat, + .size = data->stream_.configuration().size, + }; + + std::map entityControls; + entityControls[0] = data->sensor_->controls(); + + data->ipa_->configure(streamConfig, entityControls); + return ret; } void PipelineHandlerRkISP1::stop(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; ret = video_->streamOff(); @@ -372,6 +593,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera) LOG(RkISP1, Warning) << "Failed to stop camera " << camera->name(); + ret = stat_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop statistics " << camera->name(); + + ret = param_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop parameters " << camera->name(); + + data->timeline_.reset(); + activeCamera_ = nullptr; } @@ -387,12 +620,22 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request) return -ENOENT; } - int ret = video_->queueBuffer(buffer); - if (ret < 0) - return ret; - PipelineHandler::queueRequest(camera, request); + data->frameInfo_[data->frame_] = request; + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST; + op.data = { data->frame_ }; + op.controls = { request->controls() }; + data->ipa_->processEvent(op); + + data->timeline_.scheduleAction(new RkISP1ActionQueueBuffer(data->frame_, + QueueVideo, + video_, + buffer)); + data->frame_++; + return 0; } @@ -435,11 +678,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) std::unique_ptr data = utils::make_unique(this); + data->controlInfo_.emplace(std::piecewise_construct, + std::forward_as_tuple(&controls::AeEnable), + std::forward_as_tuple(false, true)); + data->sensor_ = new CameraSensor(sensor); ret = data->sensor_->init(); if (ret) return ret; + ret = data->loadIPA(); + if (ret) + return ret; + std::set streams{ &data->stream_ }; std::shared_ptr camera = Camera::create(this, sensor->name(), streams); @@ -478,7 +729,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) if (video_->open() < 0) return false; + stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics"); + if (stat_->open() < 0) + return false; + + param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params"); + if (param_->open() < 0) + return false; + video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady); + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); /* Configure default links. */ if (initLinks() < 0) { @@ -504,13 +765,52 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) * Buffer Handling */ +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request) +{ + if (request->hasPendingBuffers()) + return; + + if (request->metaData().empty()) + return; + + completeRequest(activeCamera_, request); +} + void PipelineHandlerRkISP1::bufferReady(Buffer *buffer) { ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); Request *request = buffer->request(); + data->timeline_.bufferReady(buffer); + + if (data->frame_ <= buffer->sequence()) + data->frame_ = buffer->sequence() + 1; + completeBuffer(activeCamera_, request, buffer); - completeRequest(activeCamera_, request); + tryCompleteRequest(request); +} + +void PipelineHandlerRkISP1::paramReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_PARAM, buffer->index() }; + data->ipa_->processEvent(op); +} + +void PipelineHandlerRkISP1::statReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_STAT, buffer->index() }; + data->ipa_->processEvent(op); } REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1); diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h new file mode 100644 index 0000000000000000..1972c8bbc1286e0e --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/rkisp1.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Pipeline handler for Rockchip ISP1 + */ +#ifndef __LIBCAMERA_RKISP1_H__ +#define __LIBCAMERA_RKISP1_H__ + +#include +#include + +#include "camera_sensor.h" +#include "timeline.h" +#include "v4l2_videodevice.h" + +namespace libcamera { + +enum RkISP1ActionType { + SetSensor, + SOE, + QueueVideo, + QueueParameters, + QueueStatistics, +}; + +class RkISP1ActionSetSensor : public FrameAction +{ +public: + RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls) + : FrameAction(SetSensor, frame), sensor_(sensor), controls_(controls) {} + +protected: + void run() override; + +private: + CameraSensor *sensor_; + V4L2ControlList controls_; +}; + +class RkISP1ActionQueueBuffer : public FrameAction +{ +public: + RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type, + V4L2VideoDevice *device, Buffer *buffer) + : FrameAction(type, frame), device_(device), buffer_(buffer) + { + } + +protected: + void run() override; + +private: + V4L2VideoDevice *device_; + Buffer *buffer_; +}; + +class RkISP1Timeline : public Timeline +{ +public: + RkISP1Timeline() + : Timeline() + { + setDelay(SetSensor, -1, 5); + setDelay(SOE, 0, -1); + setDelay(QueueVideo, -1, 10); + setDelay(QueueParameters, -1, 8); + setDelay(QueueStatistics, -1, 8); + } + + void bufferReady(Buffer *buffer); + + void setDelay(unsigned int type, int frame, int msdelay); +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_RKISP1_H__ */ diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp new file mode 100644 index 0000000000000000..ca0cd96711d84244 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline handler for Rockchip ISP1 + */ + +#include "rkisp1.h" + +#include "log.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(RkISP1) + +void RkISP1ActionSetSensor::run() +{ + sensor_->setControls(&controls_); +} + +void RkISP1ActionQueueBuffer::run() +{ + int ret = device_->queueBuffer(buffer_); + if (ret < 0) + LOG(RkISP1, Error) << "Failed to queue buffer"; +} + +void RkISP1Timeline::bufferReady(Buffer *buffer) +{ + /* + * Calculate SOE by taking the end of DMA set by the kernel and applying + * the time offsets provieded by the IPA to find the best estimate of + * SOE. + * + * NOTE: Make sure the IPA do not set a frame offset for the SOE action + * type as the frame interval is calculated using the interval between + * two SOE events. So using a frame interval in the SOE estimate creates + * a recursion. + */ + + ASSERT(frameOffset(SOE) == 0); + + utils::time_point soe = std::chrono::time_point() + + std::chrono::nanoseconds(buffer->timestamp()) + + timeOffset(SOE); + + notifyStartOfExposure(buffer->sequence(), soe); +} + +void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay) +{ + utils::duration delay = std::chrono::milliseconds(msdelay); + setRawDelay(type, frame, delay); +} + +} /* namespace libcamera */