From patchwork Fri Sep 27 02:44:05 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2025 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id F2F95616FF for ; Fri, 27 Sep 2019 04:45:24 +0200 (CEST) X-Halon-ID: cba09546-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cba09546-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:44:59 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:05 +0200 Message-Id: <20190927024417.725906-2-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 01/13] libcamera: pipeline: Move IPA from pipeline to camera data X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:25 -0000 The IPA acts on a camera and not on a pipeline which can expose more then one camera. Move the IPA reference to the CameraData and move the loading of an IPA from the specific pipeline handler implementation to base PipelineHandler. It's still possible to expose a camera without an IPA but if an IPA is request the camera is not valid and will not be registered in the system if a suiting IPA module can't be found. Signed-off-by: Niklas Söderlund --- src/libcamera/include/pipeline_handler.h | 4 +++ src/libcamera/pipeline/vimc.cpp | 17 ++++++------ src/libcamera/pipeline_handler.cpp | 33 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/libcamera/include/pipeline_handler.h b/src/libcamera/include/pipeline_handler.h index 1fdef9cea40f1f0a..4400c7ed835f551d 100644 --- a/src/libcamera/include/pipeline_handler.h +++ b/src/libcamera/include/pipeline_handler.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -39,10 +40,13 @@ public: } virtual ~CameraData() {} + virtual int loadIPA() { return 0; }; + Camera *camera_; PipelineHandler *pipe_; std::list queuedRequests_; ControlInfoMap controlInfo_; + std::unique_ptr ipa_; private: CameraData(const CameraData &) = delete; diff --git a/src/libcamera/pipeline/vimc.cpp b/src/libcamera/pipeline/vimc.cpp index f26a91f86ec1794c..2fa48586fea0b80e 100644 --- a/src/libcamera/pipeline/vimc.cpp +++ b/src/libcamera/pipeline/vimc.cpp @@ -52,6 +52,15 @@ public: delete raw_; } + int loadIPA() override + { + ipa_ = IPAManager::instance()->createIPA(pipe_, 0, 0); + if (!ipa_) + return -ENOENT; + + return 0; + } + int init(MediaDevice *media); void bufferReady(Buffer *buffer); @@ -100,8 +109,6 @@ private: return static_cast( PipelineHandler::cameraData(camera)); } - - std::unique_ptr ipa_; }; VimcCameraConfiguration::VimcCameraConfiguration() @@ -361,12 +368,6 @@ bool PipelineHandlerVimc::match(DeviceEnumerator *enumerator) if (!media) return false; - ipa_ = IPAManager::instance()->createIPA(this, 0, 0); - if (ipa_ == nullptr) - LOG(VIMC, Warning) << "no matching IPA found"; - else - ipa_->init(); - std::unique_ptr data = utils::make_unique(this); /* Locate and open the capture video node. */ diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index 3e54aa23d92b9a36..b8a3787e10a587b5 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -12,6 +12,7 @@ #include #include "device_enumerator.h" +#include "ipa_manager.h" #include "log.h" #include "media_device.h" #include "utils.h" @@ -58,6 +59,20 @@ LOG_DEFINE_CATEGORY(Pipeline) * exists. */ +/** + * \fn CameraData::loadIPA() + * \brief Load an IPA for the camera + * + * This function shall be implemented by pipeline handlers that wish to have an + * IPA. The function must locate and load an IPA, storing a pointer to it in + * the \a ipa_ or fail. + * + * If the pipeline handler do not wish to use an IPA this function shall not + * be implemented. + * + * \return True if a IPA could be loaded, false otherwise + */ + /** * \var CameraData::camera_ * \brief The camera related to this CameraData instance @@ -96,6 +111,14 @@ LOG_DEFINE_CATEGORY(Pipeline) * creating the camera, and shall not be modified afterwards. */ +/** + * \var CameraData::ipa_ + * \brief The IPA module used by the camera + * + * Reference to the Image Processing Algorithms (IPA) operating on the camera's + * stream(s). If no IPAs are in operation this should be set to nullptr. + */ + /** * \class PipelineHandler * \brief Create and manage cameras based on a set of media devices @@ -425,6 +448,16 @@ void PipelineHandler::registerCamera(std::shared_ptr camera, std::unique_ptr data) { data->camera_ = camera.get(); + + if (data->loadIPA()) { + LOG(Pipeline, Warning) << "Skipping " << camera->name() + << " no IPA found"; + return; + } + + if (data->ipa_) + data->ipa_->init(); + cameraData_[camera.get()] = std::move(data); cameras_.push_back(camera); manager_->addCamera(std::move(camera)); From patchwork Fri Sep 27 02:44:06 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2026 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 665E36170D for ; Fri, 27 Sep 2019 04:45:25 +0200 (CEST) X-Halon-ID: cc3ddcf6-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cc3ddcf6-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:44:59 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:06 +0200 Message-Id: <20190927024417.725906-3-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 02/13] libcamera: controls: Add AeEnable X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:25 -0000 Add a control to turn Auto Exposure on or off. Signed-off-by: Niklas Söderlund --- include/libcamera/control_ids.h | 1 + src/libcamera/controls.cpp | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/include/libcamera/control_ids.h b/include/libcamera/control_ids.h index 75b6a2d5cafeca72..8cd44e571f705ac5 100644 --- a/include/libcamera/control_ids.h +++ b/include/libcamera/control_ids.h @@ -13,6 +13,7 @@ namespace libcamera { enum ControlId { + AeEnable, AwbEnable, Brightness, Contrast, diff --git a/src/libcamera/controls.cpp b/src/libcamera/controls.cpp index 727fdbd9450d2f40..44cb09d9102d93f6 100644 --- a/src/libcamera/controls.cpp +++ b/src/libcamera/controls.cpp @@ -186,6 +186,13 @@ std::string ControlValue::toString() const * \brief Numerical control ID */ +/** + * \var AeEnable + * ControlType: Bool + * + * Enable or disable the auto-exposure algorithm. \sa ControlId::ManualExposure. + */ + /** * \var AwbEnable * ControlType: Bool @@ -218,14 +225,20 @@ std::string ControlValue::toString() const * \var ManualExposure * ControlType: Integer * - * Specify a fixed exposure time in milli-seconds + * Specify a fixed exposure time in milli-seconds. + * + * This control is only considered if AeEnable is not enabled. If present at + * at the same time as AeEnable the value will be ignored. */ /** * \var ManualGain * ControlType: Integer * - * Specify a fixed gain parameter + * Specify a fixed gain parameter. + * + * This control is only considered if AeEnable is not enabled. If present at + * at the same time as AeEnable the value will be ignored. */ /** From patchwork Fri Sep 27 02:44:07 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2027 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id EE73F61917 for ; Fri, 27 Sep 2019 04:45:26 +0200 (CEST) X-Halon-ID: cc9cfc69-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cc9cfc69-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:00 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:07 +0200 Message-Id: <20190927024417.725906-4-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 03/13] libcamera: controls: Allow read only access to control values X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:27 -0000 Allow the control values in a ControlList to be examined from a const environment. Signed-off-by: Niklas Söderlund --- include/libcamera/controls.h | 1 + src/libcamera/controls.cpp | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/include/libcamera/controls.h b/include/libcamera/controls.h index fbb3a62274c64259..eee5ef87b8fd2633 100644 --- a/include/libcamera/controls.h +++ b/include/libcamera/controls.h @@ -124,6 +124,7 @@ public: void clear() { controls_.clear(); } ControlValue &operator[](ControlId id); + const ControlValue &operator[](ControlId id) const; ControlValue &operator[](const ControlInfo *info) { return controls_[info]; } void update(const ControlList &list); diff --git a/src/libcamera/controls.cpp b/src/libcamera/controls.cpp index 44cb09d9102d93f6..6271a0bd7518dda5 100644 --- a/src/libcamera/controls.cpp +++ b/src/libcamera/controls.cpp @@ -523,6 +523,41 @@ ControlValue &ControlList::operator[](ControlId id) return controls_[&iter->second]; } +/** + * \brief Access the control specified by \a id + * \param[in] id The control ID + * + * This method returns a const reference to the control identified by \a id. If + * no such controls is present in the list, a const reference to an empty + * control value is returned. + * + * The behaviour is undefined if the control \a id is not supported by the + * camera that the ControlList refers to. + * + * \return A const reference to the value of the control identified by \a id + */ +const ControlValue &ControlList::operator[](ControlId id) const +{ + const ControlInfoMap &controls = camera_->controls(); + static ControlValue empty; + + const auto camctrl = controls.find(id); + if (camctrl == controls.end()) { + LOG(Controls, Error) + << "Camera " << camera_->name() + << " does not support control " << id; + return empty; + } + + const auto ctrl = controls_.find(&camctrl->second); + if (ctrl == controls_.end()) { + LOG(Controls, Error) << "list does not have control " << id; + return empty; + } + + return ctrl->second; +} + /** * \fn ControlList::operator[](const ControlInfo *info) * \brief Access or insert the control specified by \a info From patchwork Fri Sep 27 02:44:08 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2028 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 080C261919 for ; Fri, 27 Sep 2019 04:45:26 +0200 (CEST) X-Halon-ID: cd30da6a-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cd30da6a-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:01 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:08 +0200 Message-Id: <20190927024417.725906-5-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 04/13] libcamera: request: Allow read only access to controls X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:27 -0000 Allow the controls in a Request to be examined from a const environment. Signed-off-by: Niklas Söderlund Reviewed-by: Laurent Pinchart --- include/libcamera/request.h | 1 + src/libcamera/request.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/libcamera/request.h b/include/libcamera/request.h index 9edf1cedc054014f..5eb012e41b4a377b 100644 --- a/include/libcamera/request.h +++ b/include/libcamera/request.h @@ -37,6 +37,7 @@ public: ~Request(); ControlList &controls() { return controls_; } + const ControlList &controls() const { return controls_; } const std::map &buffers() const { return bufferMap_; } int addBuffer(std::unique_ptr buffer); Buffer *findBuffer(Stream *stream) const; diff --git a/src/libcamera/request.cpp b/src/libcamera/request.cpp index ee2158fc7a9cf0b9..ebae99b07c696512 100644 --- a/src/libcamera/request.cpp +++ b/src/libcamera/request.cpp @@ -84,6 +84,12 @@ Request::~Request() * \return A reference to the ControlList in this request */ +/** + * \fn Request::controls() const + * \brief Retrieve the request's ControlList + * \sa Request::controls() + */ + /** * \fn Request::buffers() * \brief Retrieve the request's streams to buffers map From patchwork Fri Sep 27 02:44:09 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2029 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4AE3261917 for ; Fri, 27 Sep 2019 04:45:28 +0200 (CEST) X-Halon-ID: cd96051e-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cd96051e-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:02 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:09 +0200 Message-Id: <20190927024417.725906-6-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 05/13] libcamera: request: Add IPAMetaData X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:28 -0000 Add a new structure to hold meta data coming out of the IPA. The structure will grow over time but for now only add information about the auto exposure state as it can be directly used by the rkisp1 IPA, which is capable of controlling exposure. Signed-off-by: Niklas Söderlund --- include/libcamera/request.h | 12 +++++++++++ src/libcamera/request.cpp | 42 ++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/include/libcamera/request.h b/include/libcamera/request.h index 5eb012e41b4a377b..38a6f008d53dc53a 100644 --- a/include/libcamera/request.h +++ b/include/libcamera/request.h @@ -21,6 +21,16 @@ class Buffer; class Camera; class Stream; +enum AeState { + Inactive, + Searching, + Converged, +}; + +struct IPAMetaData { + AeState aeState; + bool ready; +}; class Request { @@ -41,6 +51,7 @@ public: const std::map &buffers() const { return bufferMap_; } int addBuffer(std::unique_ptr buffer); Buffer *findBuffer(Stream *stream) const; + const IPAMetaData &metaData() const { return metaData_; }; uint64_t cookie() const { return cookie_; } Status status() const { return status_; } @@ -60,6 +71,7 @@ private: ControlList controls_; std::map bufferMap_; std::unordered_set pending_; + IPAMetaData metaData_; const uint64_t cookie_; Status status_; diff --git a/src/libcamera/request.cpp b/src/libcamera/request.cpp index ebae99b07c696512..226a10b6a3c048c5 100644 --- a/src/libcamera/request.cpp +++ b/src/libcamera/request.cpp @@ -24,6 +24,40 @@ namespace libcamera { LOG_DEFINE_CATEGORY(Request) +/** + * \enum AeState + * State of Auto Exposure algorithm + * \var AeState::Inactive + * AE not running + * \var AeState::Searching + * AE is not converged to a good value and is adjusting exposure parameters. + * \var AeState::Converged + * AE has found good exposure values for the current scene. + */ + +/** + * \struct IPAMetaData + * \brief Meta data describing the state of the IPA + * + * Container for IPA meta data. The intended creator of this object is an IPA + * and the intended consumer is applications. Applications access the object + * thru the Request object that corresponds to the specific capture event + * that generated the meta data. + */ + +/** + * \var IPAMetaData::aeState + * \brief Holds the state of the Auto Exposure algorithm + */ + +/** + * \var IPAMetaData::ready + * \brief Flag to indicate the pipeline have validated the meta data + * + * The meta data should not be returned to the application by the specific + * pipeline handler implementation before this flag is set to true. + */ + /** * \enum Request::Status * Request completion status @@ -55,7 +89,7 @@ LOG_DEFINE_CATEGORY(Request) * */ Request::Request(Camera *camera, uint64_t cookie) - : camera_(camera), controls_(camera), cookie_(cookie), + : camera_(camera), controls_(camera), metaData_({}), cookie_(cookie), status_(RequestPending), cancelled_(false) { } @@ -157,6 +191,12 @@ Buffer *Request::findBuffer(Stream *stream) const return it->second; } +/** + * \fn Request::metaData() + * \brief Retrieve the request's meta data + * \return The meta data associated with the request + */ + /** * \fn Request::cookie() * \brief Retrieve the cookie set when the request was created From patchwork Fri Sep 27 02:44:10 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2030 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D9B1B61961 for ; Fri, 27 Sep 2019 04:45:29 +0200 (CEST) X-Halon-ID: ce03760d-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id ce03760d-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:03 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:10 +0200 Message-Id: <20190927024417.725906-7-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 06/13] libcamera: pipeline: Add helper to process meta data coming from IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:30 -0000 Add a helper to process meta data coming out of an IPA and associating it with a request. The helper is needed as the pipeline handler needs to access the private meta data member of the request, something only the base pipeline class handler can do. Signed-off-by: Niklas Söderlund --- src/libcamera/include/pipeline_handler.h | 3 +++ src/libcamera/pipeline_handler.cpp | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/libcamera/include/pipeline_handler.h b/src/libcamera/include/pipeline_handler.h index 4400c7ed835f551d..d2581a327c0804c3 100644 --- a/src/libcamera/include/pipeline_handler.h +++ b/src/libcamera/include/pipeline_handler.h @@ -30,6 +30,7 @@ class DeviceMatch; class MediaDevice; class PipelineHandler; class Request; +struct IPAMetaData; class CameraData { @@ -94,6 +95,8 @@ protected: CameraData *cameraData(const Camera *camera); + void processMetaData(Request *request, const IPAMetaData &metaData); + CameraManager *manager_; private: diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index b8a3787e10a587b5..da0ad678b652768c 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -480,6 +480,22 @@ void PipelineHandler::hotplugMediaDevice(MediaDevice *media) media->disconnected.connect(this, &PipelineHandler::mediaDeviceDisconnected); } +/** + * \brief Helper to process meta data from the IPA + * \param[in] request The request to associate the \a metaData with + * \param[in] metaData The meta data to process + * + * This function is a helper for pipline handler implementations to process + * meta data retrived from an IPA. It is mandatory to call this function with + * any meta data returned from the IPA before it's passed to the application. + */ +void PipelineHandler::processMetaData(Request *request, + const IPAMetaData &metaData) +{ + request->metaData_ = metaData; + request->metaData_.ready = true; +} + /** * \brief Slot for the MediaDevice disconnected signal */ From patchwork Fri Sep 27 02:44:11 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2031 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 432ED61961 for ; Fri, 27 Sep 2019 04:45:30 +0200 (CEST) X-Halon-ID: cf0a8577-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cf0a8577-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:04 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:11 +0200 Message-Id: <20190927024417.725906-8-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 07/13] libcamera: ipa: meson: Allow access to internal libcamera headers X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:30 -0000 Allow IPA implementations to use internal libcamera headers so they can utilize helpers not exposed to applications. Signed-off-by: Niklas Söderlund Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/ipa/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipa/meson.build b/src/ipa/meson.build index f09915bc13887bd3..10448c2ffc76af4b 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -8,7 +8,7 @@ ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources ipa = shared_module(t[0], 'ipa_dummy.cpp', name_prefix : '', - include_directories : libcamera_includes, + include_directories : includes, install : true, install_dir : ipa_install_dir, cpp_args : '-DLICENSE="' + t[1] + '"') From patchwork Fri Sep 27 02:44:12 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2032 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 363C66191D for ; Fri, 27 Sep 2019 04:45:31 +0200 (CEST) X-Halon-ID: cf80c3f3-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id cf80c3f3-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:05 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:12 +0200 Message-Id: <20190927024417.725906-9-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 08/13] libcamera: ipa: Extend to support IPA interactions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:31 -0000 The IPA interface needs to support interactions with the pipeline, add interfaces to control the sensor and handling of request ISP parameters and statistics. Signed-off-by: Niklas Söderlund --- include/ipa/ipa_interface.h | 21 +++++ src/ipa/ipa_dummy.cpp | 6 +- src/libcamera/ipa_interface.cpp | 106 ++++++++++++++++++++++++ src/libcamera/proxy/ipa_proxy_linux.cpp | 13 ++- 4 files changed, 137 insertions(+), 9 deletions(-) diff --git a/include/ipa/ipa_interface.h b/include/ipa/ipa_interface.h index 2c5eb1fd524311cb..feeb00b1b1315ca5 100644 --- a/include/ipa/ipa_interface.h +++ b/include/ipa/ipa_interface.h @@ -7,14 +7,35 @@ #ifndef __LIBCAMERA_IPA_INTERFACE_H__ #define __LIBCAMERA_IPA_INTERFACE_H__ +#include +#include + +#include "v4l2_controls.h" + namespace libcamera { +class BufferMemory; +class Request; +struct IPAMetaData; + class IPAInterface { public: virtual ~IPAInterface() {} virtual int init() = 0; + + virtual void initSensor(const V4L2ControlInfoMap &controls) = 0; + virtual void initBuffers(unsigned int type, const std::vector &buffers) = 0; + + virtual void signalBuffer(unsigned int type, unsigned int id) = 0; + + virtual void queueRequest(unsigned int frame, const ControlList &controls) = 0; + + Signal updateSensor; + Signal queueBuffer; + Signal setDelay; + Signal metaDataReady; }; } /* namespace libcamera */ diff --git a/src/ipa/ipa_dummy.cpp b/src/ipa/ipa_dummy.cpp index 9d0cbdc8b1ad5565..4d429f20d7aaa955 100644 --- a/src/ipa/ipa_dummy.cpp +++ b/src/ipa/ipa_dummy.cpp @@ -15,7 +15,11 @@ namespace libcamera { class IPADummy : public IPAInterface { public: - int init(); + int init() override; + void initSensor(const V4L2ControlInfoMap &controls) override {} + void initBuffers(unsigned int type, const std::vector &buffers) override {} + void signalBuffer(unsigned int type, unsigned int id) override {} + void queueRequest(unsigned int frame, const ControlList &controls) override {} }; int IPADummy::init() diff --git a/src/libcamera/ipa_interface.cpp b/src/libcamera/ipa_interface.cpp index d7d8ca8881efcf66..79853771c70f15d4 100644 --- a/src/libcamera/ipa_interface.cpp +++ b/src/libcamera/ipa_interface.cpp @@ -24,4 +24,110 @@ namespace libcamera { * \brief Initialise the IPAInterface */ +/** + * \fn IPAInterface::initSensor() + * \brief Initialize the IPA sensor settings + * \param[in] controls List of controls provided by the sensor + * + * This function is called when a pipeline attaches to an IPA to inform the IPA + * of the controls and limits the sensor in the video pipeline supports. The IPA + * have the option to set controls of the sensor by emitting the updateSensor + * signal. + */ + +/** + * \fn IPAInterface::initBuffers() + * \brief Initialize the buffers shared between pipeline and IPA + * \param[in] type Type of buffers described in \a buffers + * \param[in] buffers List of buffers of \a type + * + * This function is called when a pipeline handler wants to inform the IPA of + * which buffers it has mapped for a specific purpose. All buffers shared + * between these two object's needs to be shared using this function prior to + * use. + * + * After the buffers have been initialized they are referenced using an + * numerical \a id which represents the insertion order in the \a buffers list + * given as an argument here. + * + * It is possible to call this function multiple times for different kinds of + * buffer types, e.g. once for statistic buffers and once more for parameter + * buffers. It's also possible to call this function multiple times for the + * same buffer type if the pipeline handler wants to update the mappings inside + * the IPA. + * + * The buffer type numerical ids and it's usage are not enforced by the IPA + * interface and is left as pipeline handler specific. + */ + +/** + * \fn IPAInterface::signalBuffer() + * \brief Signal that the pipeline handlers is done processing a buffer + * \param[in] type Type of buffer + * \param[in] id Buffer id in \a type + * + * A pipeline handler can signal to the IPA that it is done with buffer. This + * may have different meanings for different buffer types. For example signaling + * a parameter buffer may be an indication that it's now free and the IPA can + * update its content and schedule it to be written to the hardware again. While + * signaling a parameters buffer may indicate that it's filled with data from + * the hardware and are ready to be consumed by the IPA. + * + * As all pipeline designs are under the unique it must document what signaling + * a buffer of a specific type means for that particular IPA interface. + */ + +/** + * \fn IPAInterface::queueRequest() + * \brief Inform the IPA that a request have been queued to the pipeline + * \param[in] frame The frame number for the request + * \param[in] controls List of controls associated with the request + * + * This function is called by a pipeline handler when it has a request to + * process. The pipeline informs the IPA that \a frame shall be processed + * with the parameters in \a controls. + * + * The IPA may act on this by emitting different signals in the IPA interface + * to configure the pipeline to act on \a frame according to the \a controls. + */ + +/** + * \var IPAInterface::updateSensor + * \brief Signal emitted when the IPA wish to update sensor V4L2 controls + * + * This signal is emitted when the IPA wish to update one or more V4L2 control + * of the sensor in the video pipeline. The frame number the settings should be + * ready for and a list of controls to modify is passed as parameters. + */ + +/** + * \var IPAInterface::queueBuffer + * \brief Signal emitted when the IPA wish to queue a buffer to the hardware + * + * This signal is emitted then the IPA wish to queue q buffer to the hardware. + * The frame number and buffer type and id are passed as parameters. + */ + +/** + * \var IPAInterface::setDelay + * \brief Signal emitted when the IPA wish to set a delay in the pipeline + * + * This signal is emitted when the IPA wish to change a delay inside the + * pipeline timeline. The action type, frame and time offsets are passed as + * parameters. + * + * The unit for the time offset is not fixed in the IPA interface, it is up to + * the pipeline implementation to choose a suitable unit for its use-case. + */ + +/** + * \var IPAInterface::metaDataReady + * \brief Signal emitted when the IPA is done processing statistics + * + * This signal is emitted when the IPA have finished processing the statistics + * buffer and have created an IPAMetaData object which are ready to be consumed + * by the pipeline handler. The frame number and the meta data is passed as + * parameters. + */ + } /* namespace libcamera */ diff --git a/src/libcamera/proxy/ipa_proxy_linux.cpp b/src/libcamera/proxy/ipa_proxy_linux.cpp index 62fcb529e1c7e4ff..17d09fb21582376d 100644 --- a/src/libcamera/proxy/ipa_proxy_linux.cpp +++ b/src/libcamera/proxy/ipa_proxy_linux.cpp @@ -26,7 +26,11 @@ public: IPAProxyLinux(IPAModule *ipam); ~IPAProxyLinux(); - int init(); + int init() override { return 0; } + void initSensor(const V4L2ControlInfoMap &controls) override {} + void initBuffers(unsigned int type, const std::vector &buffers) override {} + void signalBuffer(unsigned int type, unsigned int id) override {} + void queueRequest(unsigned int frame, const ControlList &controls) override {} private: void readyRead(IPCUnixSocket *ipc); @@ -36,13 +40,6 @@ private: IPCUnixSocket *socket_; }; -int IPAProxyLinux::init() -{ - LOG(IPAProxy, Debug) << "initializing IPA via dummy proxy!"; - - return 0; -} - IPAProxyLinux::IPAProxyLinux(IPAModule *ipam) : proc_(nullptr), socket_(nullptr) { From patchwork Fri Sep 27 02:44:13 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2033 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id C6C4761767 for ; Fri, 27 Sep 2019 04:45:32 +0200 (CEST) X-Halon-ID: d00a3bf4-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id d00a3bf4-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:06 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:13 +0200 Message-Id: <20190927024417.725906-10-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 09/13] libcamera: timeline: Add a basic timeline implementation X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:33 -0000 The timeline is a helper for pipeline handlers to ease interacting with an IPA. The idea is that the pipeline handler runs a timeline which schedules and executes actions on the hardware. The pipeline listens to signals from the IPA which it translates into actions which are schedule on the timeline. Signed-off-by: Niklas Söderlund --- src/libcamera/include/meson.build | 1 + src/libcamera/include/timeline.h | 71 ++++++++ src/libcamera/meson.build | 1 + src/libcamera/timeline.cpp | 267 ++++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 src/libcamera/include/timeline.h create mode 100644 src/libcamera/timeline.cpp diff --git a/src/libcamera/include/meson.build b/src/libcamera/include/meson.build index 933be8543a8d5f02..fc6402b6ffb3d47c 100644 --- a/src/libcamera/include/meson.build +++ b/src/libcamera/include/meson.build @@ -16,6 +16,7 @@ libcamera_headers = files([ 'pipeline_handler.h', 'process.h', 'thread.h', + 'timeline.h', 'utils.h', 'v4l2_controls.h', 'v4l2_device.h', diff --git a/src/libcamera/include/timeline.h b/src/libcamera/include/timeline.h new file mode 100644 index 0000000000000000..519caf5a923f35a7 --- /dev/null +++ b/src/libcamera/include/timeline.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.h - Timeline for per-frame controls + */ +#ifndef __LIBCAMERA_TIMELINE_H__ +#define __LIBCAMERA_TIMELINE_H__ + +#include +#include + +#include + +#include "utils.h" + +namespace libcamera { + +class FrameAction +{ +public: + FrameAction(unsigned int type, unsigned int frame) + : type_(type), frame_(frame) {} + + virtual ~FrameAction() {} + + unsigned int type() const { return type_; } + unsigned int frame() const { return frame_; } + + virtual void run() = 0; + +private: + unsigned int type_; + unsigned int frame_; +}; + +class Timeline +{ +public: + Timeline(); + virtual ~Timeline() {} + + virtual void reset(); + virtual void scheduleAction(FrameAction *action); + virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time); + + utils::duration frameInterval() const { return frameInterval_; } + +protected: + int frameOffset(unsigned int type) const; + utils::duration timeOffset(unsigned int type) const; + + void setRawDelay(unsigned int type, int frame, utils::duration time); + +private: + static constexpr unsigned int historyDepth = 10; + + void timeout(Timer *timer); + void updateDeadline(const utils::time_point &deadline); + + std::list> history_; + std::multimap actions_; + std::map> delays_; + utils::duration frameInterval_; + + Timer timer_; +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_TIMELINE_H__ */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 755149c55c7b4f84..fef2e8619a42e053 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -28,6 +28,7 @@ libcamera_sources = files([ 'signal.cpp', 'stream.cpp', 'thread.cpp', + 'timeline.cpp', 'timer.cpp', 'utils.cpp', 'v4l2_controls.cpp', diff --git a/src/libcamera/timeline.cpp b/src/libcamera/timeline.cpp new file mode 100644 index 0000000000000000..b5a713abbf3235eb --- /dev/null +++ b/src/libcamera/timeline.cpp @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline for per-frame controls + */ + +#include "timeline.h" + +#include "log.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Timeline) + +/** + * \class FrameAction + * \brief Action which can be schedule within a frames lifetime + * + * A frame action is a event which takes place at some point during a frames + * lifetime inside libcamera. Each action have two primal attributes; type and + * frame number. + * + * The type is a numerical ID which identifies the action within the pipeline + * handler. The type is pipeline specific and have no meaning to anyone outside + * the specific implementation. The frame number is the frame the action applies + * to. + */ + +/** + * \fn FrameAction::FrameAction + * \brief Create a frame action + * \param[in] type Action type + * \param[in] frame Frame number + */ + +/** + * \fn FrameAction::type() + * \brief Retrieve the type of action + * \return Action type + */ + +/** + * \fn FrameAction::frame() + * \brief Retrieve the frame number + * \return Frame number + */ + +/** + * \fn FrameAction::run() + * \brief The action to perform when the action is triggered + * + * Pipeline handlers shall subclass the FrameAction object and implement run() + * functions which describes the actions they wish to happen when the act is + * schedule using the Timeline. + * + * \sa Timeline + */ + +/** + * \class Timeline + * \brief Scheduler and executor of FrameAction's + * + * On the timeline the pipeline can schedule FrameActions and expect them to be + * executed at the correct time. The correct time to execute them are a sum + * of which frame the action should apply to and a list of timing delays for + * each action the pipeline handler describes. + * + * The timing delays can either be set by the pipeline handler or the IPA. The + * exact implementation of how the timing delays are setup are pipeline + * specific. It is expected that the IPA will update the timing delays as it + * make changes to how the pipeline operates in different situations. + * + * The pipeline handler is responsible for feeding the timeline as accurate as + * possible information of the exposures it observes. + */ + +Timeline::Timeline() + : frameInterval_(std::chrono::milliseconds(1)) +{ + timer_.timeout.connect(this, &Timeline::timeout); +} + +/** + * \brief Reset the timeline + */ +void Timeline::reset() +{ + timer_.stop(); + + actions_.clear(); + history_.clear(); +} + +/** + * \brief Add a action to the timeline + * \param[in] action FrameAction to add + * + * Schedule an action at a specific time taking the action and timing delays + * into account. If a action is schedule too late execute it immediately and + * try to recover. + */ +void Timeline::scheduleAction(FrameAction *action) +{ + unsigned int lastFrame; + utils::time_point lastTime; + + if (history_.empty()) { + /* + * This only happens for the first number of frames, up to + * pipeline depth. + */ + lastFrame = 0; + lastTime = std::chrono::steady_clock::now(); + } else { + lastFrame = history_.back().first; + lastTime = history_.back().second; + } + + /* + * Calculate action trigger time by first figuring the out the start of + * exposure (SOE) of the frame the action corresponds to and then adding + * the frame and timing offsets. + */ + int frame = action->frame() + frameOffset(action->type()) - lastFrame; + utils::time_point deadline = lastTime + frame * frameInterval_ + timeOffset(action->type()); + + utils::time_point now = std::chrono::steady_clock::now(); + if (deadline < now) { + LOG(Timeline, Warning) + << "Action schedule too late " + << utils::time_point_to_string(deadline) + << ", run now " << utils::time_point_to_string(now); + action->run(); + } else { + actions_.insert({ deadline, action }); + updateDeadline(deadline); + } +} + +/** + * \brief Inform timeline of a new exposure + * \param[in] frame Frame number of the exposure + * \param[in] time The best approximation of when the exposure started + */ +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time) +{ + history_.push_back(std::make_pair(frame, time)); + + if (history_.size() <= historyDepth / 2) + return; + + while (history_.size() > historyDepth) + history_.pop_front(); + + /* Update esitmated time between two start of exposures. */ + utils::duration sumExposures = std::chrono::duration_values::zero(); + unsigned int numExposures = 0; + + utils::time_point lastTime; + for (auto it = history_.begin(); it != history_.end(); it++) { + if (it != history_.begin()) { + sumExposures += it->second - lastTime; + numExposures++; + } + + lastTime = it->second; + } + + frameInterval_ = sumExposures; + if (numExposures) + frameInterval_ /= numExposures; +} + +/** + * \fn Timeline::frameInterval() + * \brief Retrieve the best estimate of the frame interval + * \return Frame interval estimate + */ + +/** + * \brief Retrieve the frame offset for an action type + * \param[in] type Action type + * \return Frame offset for the action type + */ +int Timeline::frameOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No frame offset set for action type " << type; + return 0; + } + + return it->second.first; +} + +/** + * \brief Retrieve the time offset for an action type + * \param[in] type Action type + * \return Time offset for the action type + */ +utils::duration Timeline::timeOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No time offset set for action type " << type; + return utils::duration::zero(); + } + + return it->second.second; +} + +/** + * \brief Set the frame and time offset delays for an action type + * \param[in] type Action type + * \param[in] frame Frame offset + * \param[in] time Time offset + */ +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time) +{ + delays_[type] = std::make_pair(frame, time); +} + +void Timeline::updateDeadline(const utils::time_point &deadline) +{ + if (timer_.isRunning() && deadline >= timer_.deadline()) + return; + + utils::time_point now = std::chrono::steady_clock::now(); + utils::duration duration = deadline - now; + + /* + * Translate nanoseconds resolution the timeline to milliseconds using + * a ceiling approach to not trigger an action before it's scheduled. + * + * \todo: Should the Timer operate using nanoseconds? + */ + unsigned long nsecs = std::chrono::duration_cast(duration).count(); + unsigned int msecs = nsecs / 1000000 + (nsecs % 1000000 ? 1 : 0); + + timer_.stop(); + timer_.start(msecs); +} + +void Timeline::timeout(Timer *timer) +{ + for (auto it = actions_.begin(); it != actions_.end();) { + utils::time_point now = std::chrono::steady_clock::now(); + + utils::time_point sched = it->first; + FrameAction *action = it->second; + + if (sched >= now) { + updateDeadline(sched); + break; + } + + action->run(); + + it = actions_.erase(it); + delete action; + } +} + +} /* namespace libcamera */ From patchwork Fri Sep 27 02:44:14 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2034 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 39BB861767 for ; Fri, 27 Sep 2019 04:45:33 +0200 (CEST) X-Halon-ID: d0cab3ad-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id d0cab3ad-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:07 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:14 +0200 Message-Id: <20190927024417.725906-11-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 10/13] test: Add timeline test X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:33 -0000 Add a test of the timeline to make sure events fire and the estimated frame interval are OK. Signed-off-by: Niklas Söderlund --- test/meson.build | 1 + test/timeline.cpp | 97 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 test/timeline.cpp diff --git a/test/meson.build b/test/meson.build index 84722cceb35db86e..6a4c4cd373043469 100644 --- a/test/meson.build +++ b/test/meson.build @@ -28,6 +28,7 @@ internal_tests = [ ['object-invoke', 'object-invoke.cpp'], ['signal-threads', 'signal-threads.cpp'], ['threads', 'threads.cpp'], + ['timeline', 'timeline.cpp'], ['timer', 'timer.cpp'], ['timer-thread', 'timer-thread.cpp'], ] diff --git a/test/timeline.cpp b/test/timeline.cpp new file mode 100644 index 0000000000000000..55b651b455a07ffc --- /dev/null +++ b/test/timeline.cpp @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline test + */ + +#include +#include +#include + +#include + +#include "test.h" +#include "thread.h" +#include "timeline.h" + +using namespace std; +using namespace libcamera; + +enum TimelineTestActionType { + Inc, +}; + +class IncrementAction : public FrameAction +{ +public: + IncrementAction(unsigned int frame, unsigned int *counter) + : FrameAction(Inc, frame), counter_(counter) + { + } + + void run() override + { + (*counter_)++; + } + +private: + unsigned int *counter_; +}; + +class TimelineTest : public Test +{ +public: + void fakeSOE() + { + timeline_.notifyStartOfExposure(sequence_++, + std::chrono::steady_clock::now()); + } + + int init() + { + sequence_ = 0; + return 0; + } + + int run() + { + EventDispatcher *dispatcher = Thread::current()->eventDispatcher(); + unsigned int loops, counter; + Timer timer; + + /* + * Run frames and make sure an event is fired for each and that + * the messured frame interval is true. + */ + loops = 500; + counter = 0; + for (unsigned int i = 0; i < loops; i++) { + timeline_.scheduleAction(new IncrementAction(i + 1, &counter)); + fakeSOE(); + timer.start(10); + while (timer.isRunning()) + dispatcher->processEvents(); + } + + cout << "Got " << counter << " evens, expected " << loops - 1 << endl; + if (counter != loops - 1) + return TestFail; + + unsigned int interval = std::chrono::duration_cast(timeline_.frameInterval()).count(); + cout << "Messured frame interval " << interval << " expected 10" << endl; + if (interval != 10) + return TestFail; + + return TestPass; + } + + void cleanup() + { + } +private: + unsigned int sequence_; + Timeline timeline_; +}; + +TEST_REGISTER(TimelineTest) From patchwork Fri Sep 27 02:44:15 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2036 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CB36C61922 for ; Fri, 27 Sep 2019 04:45:35 +0200 (CEST) X-Halon-ID: d12c36b4-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id d12c36b4-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:08 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:15 +0200 Message-Id: <20190927024417.725906-12-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 11/13] include: linux: Add rkisp1 kernel header and format definitions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:36 -0000 Add kernel header and format definitions from v8 of the rkisp1 series [1]. The driver is currently out of tree, the header file is not exported as part of the standard kernel uAPI. Headers have been exported on top of a v5.2 kernel tree using scripts/headers_install.sh. 1. https://lore.kernel.org/linux-media/20190730184256.30338-1-helen.koike@collabora.com/ Signed-off-by: Niklas Söderlund Acked-by: Kieran Bingham Acked-by: Laurent Pinchart --- include/linux/rkisp1-config.h | 816 ++++++++++++++++++++++++++++++++++ include/linux/videodev2.h | 4 + 2 files changed, 820 insertions(+) create mode 100644 include/linux/rkisp1-config.h diff --git a/include/linux/rkisp1-config.h b/include/linux/rkisp1-config.h new file mode 100644 index 0000000000000000..f0d1bd82e857fd79 --- /dev/null +++ b/include/linux/rkisp1-config.h @@ -0,0 +1,816 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip isp1 driver + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +/* + * TODO: Improve documentation, mostly regarding abbreviation and hardware + * specificities. + */ + +#ifndef _RKISP1_CONFIG_H +#define _RKISP1_CONFIG_H + +#include +#include + +#define CIFISP_MODULE_DPCC (1 << 0) +#define CIFISP_MODULE_BLS (1 << 1) +#define CIFISP_MODULE_SDG (1 << 2) +#define CIFISP_MODULE_HST (1 << 3) +#define CIFISP_MODULE_LSC (1 << 4) +#define CIFISP_MODULE_AWB_GAIN (1 << 5) +#define CIFISP_MODULE_FLT (1 << 6) +#define CIFISP_MODULE_BDM (1 << 7) +#define CIFISP_MODULE_CTK (1 << 8) +#define CIFISP_MODULE_GOC (1 << 9) +#define CIFISP_MODULE_CPROC (1 << 10) +#define CIFISP_MODULE_AFC (1 << 11) +#define CIFISP_MODULE_AWB (1 << 12) +#define CIFISP_MODULE_IE (1 << 13) +#define CIFISP_MODULE_AEC (1 << 14) +#define CIFISP_MODULE_WDR (1 << 15) +#define CIFISP_MODULE_DPF (1 << 16) +#define CIFISP_MODULE_DPF_STRENGTH (1 << 17) + +#define CIFISP_CTK_COEFF_MAX 0x100 +#define CIFISP_CTK_OFFSET_MAX 0x800 + +#define CIFISP_AE_MEAN_MAX 25 +#define CIFISP_HIST_BIN_N_MAX 16 +#define CIFISP_AFM_MAX_WINDOWS 3 +#define CIFISP_DEGAMMA_CURVE_SIZE 17 + +#define CIFISP_BDM_MAX_TH 0xFF + +/* + * Black level compensation + */ +/* maximum value for horizontal start address */ +#define CIFISP_BLS_START_H_MAX 0x00000FFF +/* maximum value for horizontal stop address */ +#define CIFISP_BLS_STOP_H_MAX 0x00000FFF +/* maximum value for vertical start address */ +#define CIFISP_BLS_START_V_MAX 0x00000FFF +/* maximum value for vertical stop address */ +#define CIFISP_BLS_STOP_V_MAX 0x00000FFF +/* maximum is 2^18 = 262144*/ +#define CIFISP_BLS_SAMPLES_MAX 0x00000012 +/* maximum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MAX 0x00000FFF +/* minimum value for fixed black level */ +#define CIFISP_BLS_FIX_SUB_MIN 0xFFFFF000 +/* 13 bit range (signed)*/ +#define CIFISP_BLS_FIX_MASK 0x00001FFF + +/* + * Automatic white balance measurments + */ +#define CIFISP_AWB_MAX_GRID 1 +#define CIFISP_AWB_MAX_FRAMES 7 + +/* + * Gamma out + */ +/* Maximum number of color samples supported */ +#define CIFISP_GAMMA_OUT_MAX_SAMPLES 17 + +/* + * Lens shade correction + */ +#define CIFISP_LSC_GRAD_TBL_SIZE 8 +#define CIFISP_LSC_SIZE_TBL_SIZE 8 +/* + * The following matches the tuning process, + * not the max capabilities of the chip. + * Last value unused. + */ +#define CIFISP_LSC_DATA_TBL_SIZE 290 + +/* + * Histogram calculation + */ +/* Last 3 values unused. */ +#define CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE 28 + +/* + * Defect Pixel Cluster Correction + */ +#define CIFISP_DPCC_METHODS_MAX 3 + +/* + * Denoising pre filter + */ +#define CIFISP_DPF_MAX_NLF_COEFFS 17 +#define CIFISP_DPF_MAX_SPATIAL_COEFFS 6 + +/* + * Measurement types + */ +#define CIFISP_STAT_AWB (1 << 0) +#define CIFISP_STAT_AUTOEXP (1 << 1) +#define CIFISP_STAT_AFM_FIN (1 << 2) +#define CIFISP_STAT_HIST (1 << 3) + +enum cifisp_histogram_mode { + CIFISP_HISTOGRAM_MODE_DISABLE, + CIFISP_HISTOGRAM_MODE_RGB_COMBINED, + CIFISP_HISTOGRAM_MODE_R_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_G_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_B_HISTOGRAM, + CIFISP_HISTOGRAM_MODE_Y_HISTOGRAM +}; + +enum cifisp_awb_mode_type { + CIFISP_AWB_MODE_MANUAL, + CIFISP_AWB_MODE_RGB, + CIFISP_AWB_MODE_YCBCR +}; + +enum cifisp_flt_mode { + CIFISP_FLT_STATIC_MODE, + CIFISP_FLT_DYNAMIC_MODE +}; + +/** + * enum cifisp_exp_ctrl_autostop - stop modes + * @CIFISP_EXP_CTRL_AUTOSTOP_0: continuous measurement + * @CIFISP_EXP_CTRL_AUTOSTOP_1: stop measuring after a complete frame + */ +enum cifisp_exp_ctrl_autostop { + CIFISP_EXP_CTRL_AUTOSTOP_0 = 0, + CIFISP_EXP_CTRL_AUTOSTOP_1 = 1, +}; + +/** + * enum cifisp_exp_meas_mode - Exposure measure mode + * @CIFISP_EXP_MEASURING_MODE_0: Y = 16 + 0.25R + 0.5G + 0.1094B + * @CIFISP_EXP_MEASURING_MODE_1: Y = (R + G + B) x (85/256) + */ +enum cifisp_exp_meas_mode { + CIFISP_EXP_MEASURING_MODE_0, + CIFISP_EXP_MEASURING_MODE_1, +}; + +/*---------- PART1: Input Parameters ------------*/ + +struct cifisp_window { + __u16 h_offs; + __u16 v_offs; + __u16 h_size; + __u16 v_size; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_fixed_val - BLS fixed subtraction values + * + * The values will be subtracted from the sensor + * values. Therefore a negative value means addition instead of subtraction! + * + * @r: Fixed (signed!) subtraction value for Bayer pattern R + * @gr: Fixed (signed!) subtraction value for Bayer pattern Gr + * @gb: Fixed (signed!) subtraction value for Bayer pattern Gb + * @b: Fixed (signed!) subtraction value for Bayer pattern B + */ +struct cifisp_bls_fixed_val { + __s16 r; + __s16 gr; + __s16 gb; + __s16 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_config - Configuration used by black level subtraction + * + * @enable_auto: Automatic mode activated means that the measured values + * are subtracted. Otherwise the fixed subtraction + * values will be subtracted. + * @en_windows: enabled window + * @bls_window1: Measurement window 1 size + * @bls_window2: Measurement window 2 size + * @bls_samples: Set amount of measured pixels for each Bayer position + * (A, B,C and D) to 2^bls_samples. + * @cifisp_bls_fixed_val: Fixed subtraction values + */ +struct cifisp_bls_config { + __u8 enable_auto; + __u8 en_windows; + struct cifisp_window bls_window1; + struct cifisp_window bls_window2; + __u8 bls_samples; + struct cifisp_bls_fixed_val fixed_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Methods Configuration used by DPCC + * + * Methods Configuration used by Defect Pixel Cluster Correction + * + * @method: Method enable bits + * @line_thresh: Line threshold + * @line_mad_fac: Line MAD factor + * @pg_fac: Peak gradient factor + * @rnd_thresh: Rank Neighbor Difference threshold + * @rg_fac: Rank gradient factor + */ +struct cifisp_dpcc_methods_config { + __u32 method; + __u32 line_thresh; + __u32 line_mad_fac; + __u32 pg_fac; + __u32 rnd_thresh; + __u32 rg_fac; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpcc_methods_config - Configuration used by DPCC + * + * Configuration used by Defect Pixel Cluster Correction + * + * @mode: dpcc output mode + * @output_mode: whether use hard coded methods + * @set_use: stage1 methods set + * @methods: methods config + * @ro_limits: rank order limits + * @rnd_offs: differential rank offsets for rank neighbor difference + */ +struct cifisp_dpcc_config { + __u32 mode; + __u32 output_mode; + __u32 set_use; + struct cifisp_dpcc_methods_config methods[CIFISP_DPCC_METHODS_MAX]; + __u32 ro_limits; + __u32 rnd_offs; +} __attribute__ ((packed)); + +struct cifisp_gamma_corr_curve { + __u16 gamma_y[CIFISP_DEGAMMA_CURVE_SIZE]; +} __attribute__ ((packed)); + +struct cifisp_gamma_curve_x_axis_pnts { + __u32 gamma_dx0; + __u32 gamma_dx1; +} __attribute__ ((packed)); + +/** + * struct cifisp_gamma_corr_curve - Configuration used by sensor degamma + * + * @curve_x: gamma curve point definition axis for x + * @xa_pnts: x increments + */ +struct cifisp_sdg_config { + struct cifisp_gamma_corr_curve curve_r; + struct cifisp_gamma_corr_curve curve_g; + struct cifisp_gamma_corr_curve curve_b; + struct cifisp_gamma_curve_x_axis_pnts xa_pnts; +} __attribute__ ((packed)); + +/** + * struct cifisp_lsc_config - Configuration used by Lens shading correction + * + * refer to REF_01 for details + */ +struct cifisp_lsc_config { + __u32 r_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gr_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 gb_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + __u32 b_data_tbl[CIFISP_LSC_DATA_TBL_SIZE]; + + __u32 x_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + __u32 y_grad_tbl[CIFISP_LSC_GRAD_TBL_SIZE]; + + __u32 x_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u32 y_size_tbl[CIFISP_LSC_SIZE_TBL_SIZE]; + __u16 config_width; + __u16 config_height; +} __attribute__ ((packed)); + +/** + * struct cifisp_ie_config - Configuration used by image effects + * + * @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1 + * @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2 + * @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1 + * @eff_mat_4: 3x3 Matrix Coefficients for Sketch Effect 2 + * @eff_mat_5: 3x3 Matrix Coefficients for Sketch Effect 3 + * @eff_tint: Chrominance increment values of tint (used for sepia effect) + */ +struct cifisp_ie_config { + __u16 effect; + __u16 color_sel; + __u16 eff_mat_1; + __u16 eff_mat_2; + __u16 eff_mat_3; + __u16 eff_mat_4; + __u16 eff_mat_5; + __u16 eff_tint; +} __attribute__ ((packed)); + +/** + * struct cifisp_cproc_config - Configuration used by Color Processing + * + * @c_out_range: Chrominance pixel clipping range at output. + * (0 for limit, 1 for full) + * @y_in_range: Luminance pixel clipping range at output. + * @y_out_range: Luminance pixel clipping range at output. + * @contrast: 00~ff, 0.0~1.992 + * @brightness: 80~7F, -128~+127 + * @sat: saturation, 00~FF, 0.0~1.992 + * @hue: 80~7F, -90~+87.188 + */ +struct cifisp_cproc_config { + __u8 c_out_range; + __u8 y_in_range; + __u8 y_out_range; + __u8 contrast; + __u8 brightness; + __u8 sat; + __u8 hue; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_meas_config - Configuration used by auto white balance + * + * @awb_wnd: white balance measurement window (in pixels) + * (from enum cifisp_awb_mode_type) + * @max_y: only pixels values < max_y contribute to awb measurement, set to 0 + * to disable this feature + * @min_y: only pixels values > min_y contribute to awb measurement + * @max_csum: Chrominance sum maximum value, only consider pixels with Cb+Cr, + * smaller than threshold for awb measurements + * @min_c: Chrominance minimum value, only consider pixels with Cb/Cr + * each greater than threshold value for awb measurements + * @frames: number of frames - 1 used for mean value calculation + * (ucFrames=0 means 1 Frame) + * @awb_ref_cr: reference Cr value for AWB regulation, target for AWB + * @awb_ref_cb: reference Cb value for AWB regulation, target for AWB + */ +struct cifisp_awb_meas_config { + /* + * Note: currently the h and v offsets are mapped to grid offsets + */ + struct cifisp_window awb_wnd; + __u32 awb_mode; + __u8 max_y; + __u8 min_y; + __u8 max_csum; + __u8 min_c; + __u8 frames; + __u8 awb_ref_cr; + __u8 awb_ref_cb; + __u8 enable_ymax_cmp; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_gain_config - Configuration used by auto white balance gain + * + * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8 + */ +struct cifisp_awb_gain_config { + __u16 gain_red; + __u16 gain_green_r; + __u16 gain_blue; + __u16 gain_green_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_flt_config - Configuration used by ISP filtering + * + * @mode: ISP_FILT_MODE register fields (from enum cifisp_flt_mode) + * @grn_stage1: ISP_FILT_MODE register fields + * @chr_h_mode: ISP_FILT_MODE register fields + * @chr_v_mode: ISP_FILT_MODE register fields + * + * refer to REF_01 for details. + */ + +struct cifisp_flt_config { + __u32 mode; + __u8 grn_stage1; + __u8 chr_h_mode; + __u8 chr_v_mode; + __u32 thresh_bl0; + __u32 thresh_bl1; + __u32 thresh_sh0; + __u32 thresh_sh1; + __u32 lum_weight; + __u32 fac_sh1; + __u32 fac_sh0; + __u32 fac_mid; + __u32 fac_bl0; + __u32 fac_bl1; +} __attribute__ ((packed)); + +/** + * struct cifisp_bdm_config - Configuration used by Bayer DeMosaic + * + * @demosaic_th: threshod for bayer demosaicing texture detection + */ +struct cifisp_bdm_config { + __u8 demosaic_th; +} __attribute__ ((packed)); + +/** + * struct cifisp_ctk_config - Configuration used by Cross Talk correction + * + * @coeff: color correction matrix + * @ct_offset_b: offset for the crosstalk correction matrix + */ +struct cifisp_ctk_config { + __u16 coeff0; + __u16 coeff1; + __u16 coeff2; + __u16 coeff3; + __u16 coeff4; + __u16 coeff5; + __u16 coeff6; + __u16 coeff7; + __u16 coeff8; + __u16 ct_offset_r; + __u16 ct_offset_g; + __u16 ct_offset_b; +} __attribute__ ((packed)); + +enum cifisp_goc_mode { + CIFISP_GOC_MODE_LOGARITHMIC, + CIFISP_GOC_MODE_EQUIDISTANT +}; + +/** + * struct cifisp_goc_config - Configuration used by Gamma Out correction + * + * @mode: goc mode (from enum cifisp_goc_mode) + * @gamma_y: gamma out curve y-axis for all color components + */ +struct cifisp_goc_config { + __u32 mode; + __u16 gamma_y[CIFISP_GAMMA_OUT_MAX_SAMPLES]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hst_config - Configuration used by Histogram + * + * @mode: histogram mode (from enum cifisp_histogram_mode) + * @histogram_predivider: process every stepsize pixel, all other pixels are + * skipped + * @meas_window: coordinates of the measure window + * @hist_weight: weighting factor for sub-windows + */ +struct cifisp_hst_config { + __u32 mode; + __u8 histogram_predivider; + struct cifisp_window meas_window; + __u8 hist_weight[CIFISP_HISTOGRAM_WEIGHT_GRIDS_SIZE]; +} __attribute__ ((packed)); + +/** + * struct cifisp_aec_config - Configuration used by Auto Exposure Control + * + * @mode: Exposure measure mode (from enum cifisp_exp_meas_mode) + * @autostop: stop mode (from enum cifisp_exp_ctrl_autostop) + * @meas_window: coordinates of the measure window + */ +struct cifisp_aec_config { + __u32 mode; + __u32 autostop; + struct cifisp_window meas_window; +} __attribute__ ((packed)); + +/** + * struct cifisp_afc_config - Configuration used by Auto Focus Control + * + * @num_afm_win: max CIFISP_AFM_MAX_WINDOWS + * @afm_win: coordinates of the meas window + * @thres: threshold used for minimizing the influence of noise + * @var_shift: the number of bits for the shift operation at the end of the + * calculation chain. + */ +struct cifisp_afc_config { + __u8 num_afm_win; + struct cifisp_window afm_win[CIFISP_AFM_MAX_WINDOWS]; + __u32 thres; + __u32 var_shift; +} __attribute__ ((packed)); + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_GAIN_USAGE_DISABLED: don't use any gains in preprocessing stage + * @CIFISP_DPF_GAIN_USAGE_NF_GAINS: use only the noise function gains from + * registers DPF_NF_GAIN_R, ... + * @CIFISP_DPF_GAIN_USAGE_LSC_GAINS: use only the gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS: use the noise function gains and the + * gains from LSC module + * @CIFISP_DPF_GAIN_USAGE_AWB_GAINS: use only the gains from AWB module + * @CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: use the gains from AWB and LSC module + * @CIFISP_DPF_GAIN_USAGE_MAX: upper border (only for an internal evaluation) + */ +enum cifisp_dpf_gain_usage { + CIFISP_DPF_GAIN_USAGE_DISABLED, + CIFISP_DPF_GAIN_USAGE_NF_GAINS, + CIFISP_DPF_GAIN_USAGE_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_NF_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_GAINS, + CIFISP_DPF_GAIN_USAGE_AWB_LSC_GAINS, + CIFISP_DPF_GAIN_USAGE_MAX +}; + +/** + * enum cifisp_dpf_gain_usage - dpf gain usage + * @CIFISP_DPF_RB_FILTERSIZE_13x9: red and blue filter kernel size 13x9 + * (means 7x5 active pixel) + * @CIFISP_DPF_RB_FILTERSIZE_9x9: red and blue filter kernel size 9x9 + * (means 5x5 active pixel) + */ +enum cifisp_dpf_rb_filtersize { + CIFISP_DPF_RB_FILTERSIZE_13x9, + CIFISP_DPF_RB_FILTERSIZE_9x9, +}; + +/** + * enum cifisp_dpf_nll_scale_mode - dpf noise level scale mode + * @CIFISP_NLL_SCALE_LINEAR: use a linear scaling + * @CIFISP_NLL_SCALE_LOGARITHMIC: use a logarithmic scaling + */ +enum cifisp_dpf_nll_scale_mode { + CIFISP_NLL_SCALE_LINEAR, + CIFISP_NLL_SCALE_LOGARITHMIC, +}; + +/** + * struct cifisp_dpf_nll - Noise level lookup + * + * @coeff: Noise level Lookup coefficient + * @scale_mode: dpf noise level scale mode (from enum cifisp_dpf_nll_scale_mode) + */ +struct cifisp_dpf_nll { + __u16 coeff[CIFISP_DPF_MAX_NLF_COEFFS]; + __u32 scale_mode; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_rb_flt - Red blue filter config + * + * @fltsize: The filter size for the red and blue pixels + * (from enum cifisp_dpf_rb_filtersize) + * @spatial_coeff: Spatial weights + * @r_enable: enable filter processing for red pixels + * @b_enable: enable filter processing for blue pixels + */ +struct cifisp_dpf_rb_flt { + __u32 fltsize; + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 r_enable; + __u8 b_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_g_flt - Green filter Configuration + * + * @spatial_coeff: Spatial weights + * @gr_enable: enable filter processing for green pixels in green/red lines + * @gb_enable: enable filter processing for green pixels in green/blue lines + */ +struct cifisp_dpf_g_flt { + __u8 spatial_coeff[CIFISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 gr_enable; + __u8 gb_enable; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_gain - Noise function Configuration + * + * @mode: dpf gain usage (from enum cifisp_dpf_gain_usage) + * @nf_r_gain: Noise function Gain that replaces the AWB gain for red pixels + * @nf_b_gain: Noise function Gain that replaces the AWB gain for blue pixels + * @nf_gr_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a red line + * @nf_gb_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a blue line + */ +struct cifisp_dpf_gain { + __u32 mode; + __u16 nf_r_gain; + __u16 nf_b_gain; + __u16 nf_gr_gain; + __u16 nf_gb_gain; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_config - Configuration used by De-noising pre-filter + * + * @gain: noise function gain + * @g_flt: green filter config + * @rb_flt: red blue filter config + * @nll: noise level lookup + */ +struct cifisp_dpf_config { + struct cifisp_dpf_gain gain; + struct cifisp_dpf_g_flt g_flt; + struct cifisp_dpf_rb_flt rb_flt; + struct cifisp_dpf_nll nll; +} __attribute__ ((packed)); + +/** + * struct cifisp_dpf_strength_config - strength of the filter + * + * @r: filter strength of the RED filter + * @g: filter strength of the GREEN filter + * @b: filter strength of the BLUE filter + */ +struct cifisp_dpf_strength_config { + __u8 r; + __u8 g; + __u8 b; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_other_cfg - Parameters for some blocks in rockchip isp1 + * + * @dpcc_config: Defect Pixel Cluster Correction config + * @bls_config: Black Level Subtraction config + * @sdg_config: sensor degamma config + * @lsc_config: Lens Shade config + * @awb_gain_config: Auto White balance gain config + * @flt_config: filter config + * @bdm_config: demosaic config + * @ctk_config: cross talk config + * @goc_config: gamma out config + * @bls_config: black level subtraction config + * @dpf_config: De-noising pre-filter config + * @dpf_strength_config: dpf strength config + * @cproc_config: color process config + * @ie_config: image effects config + */ +struct cifisp_isp_other_cfg { + struct cifisp_dpcc_config dpcc_config; + struct cifisp_bls_config bls_config; + struct cifisp_sdg_config sdg_config; + struct cifisp_lsc_config lsc_config; + struct cifisp_awb_gain_config awb_gain_config; + struct cifisp_flt_config flt_config; + struct cifisp_bdm_config bdm_config; + struct cifisp_ctk_config ctk_config; + struct cifisp_goc_config goc_config; + struct cifisp_dpf_config dpf_config; + struct cifisp_dpf_strength_config dpf_strength_config; + struct cifisp_cproc_config cproc_config; + struct cifisp_ie_config ie_config; +} __attribute__ ((packed)); + +/** + * struct cifisp_isp_meas_cfg - Rockchip ISP1 Measure Parameters + * + * @awb_meas_config: auto white balance config + * @hst_config: histogram config + * @aec_config: auto exposure config + * @afc_config: auto focus config + */ +struct cifisp_isp_meas_cfg { + struct cifisp_awb_meas_config awb_meas_config; + struct cifisp_hst_config hst_config; + struct cifisp_aec_config aec_config; + struct cifisp_afc_config afc_config; +} __attribute__ ((packed)); + +/** + * struct rkisp1_isp_params_cfg - Rockchip ISP1 Input Parameters Meta Data + * + * @module_en_update: mask the enable bits of which module should be updated + * @module_ens: mask the enable value of each module, only update the module + * which correspond bit was set in module_en_update + * @module_cfg_update: mask the config bits of which module should be updated + * @meas: measurement config + * @others: other config + */ +struct rkisp1_isp_params_cfg { + __u32 module_en_update; + __u32 module_ens; + __u32 module_cfg_update; + + struct cifisp_isp_meas_cfg meas; + struct cifisp_isp_other_cfg others; +} __attribute__ ((packed)); + +/*---------- PART2: Measurement Statistics ------------*/ + +/** + * struct cifisp_bls_meas_val - AWB measured values + * + * @cnt: White pixel count, number of "white pixels" found during laster + * measurement + * @mean_y_or_g: Mean value of Y within window and frames, + * Green if RGB is selected. + * @mean_cb_or_b: Mean value of Cb within window and frames, + * Blue if RGB is selected. + * @mean_cr_or_r: Mean value of Cr within window and frames, + * Red if RGB is selected. + */ +struct cifisp_awb_meas { + __u32 cnt; + __u8 mean_y_or_g; + __u8 mean_cb_or_b; + __u8 mean_cr_or_r; +} __attribute__ ((packed)); + +/** + * struct cifisp_awb_stat - statistics automatic white balance data + * + * @awb_mean: Mean measured data + */ +struct cifisp_awb_stat { + struct cifisp_awb_meas awb_mean[CIFISP_AWB_MAX_GRID]; +} __attribute__ ((packed)); + +/** + * struct cifisp_bls_meas_val - BLS measured values + * + * @meas_r: Mean measured value for Bayer pattern R + * @meas_gr: Mean measured value for Bayer pattern Gr + * @meas_gb: Mean measured value for Bayer pattern Gb + * @meas_b: Mean measured value for Bayer pattern B + */ +struct cifisp_bls_meas_val { + __u16 meas_r; + __u16 meas_gr; + __u16 meas_gb; + __u16 meas_b; +} __attribute__ ((packed)); + +/** + * struct cifisp_ae_stat - statistics auto exposure data + * + * @exp_mean: Mean luminance value of block xx + * @bls_val: BLS measured values + * + * Image is divided into 5x5 blocks. + */ +struct cifisp_ae_stat { + __u8 exp_mean[CIFISP_AE_MEAN_MAX]; + struct cifisp_bls_meas_val bls_val; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_meas_val - AF measured values + * + * @sum: sharpness, refer to REF_01 for definition + * @lum: luminance, refer to REF_01 for definition + */ +struct cifisp_af_meas_val { + __u32 sum; + __u32 lum; +} __attribute__ ((packed)); + +/** + * struct cifisp_af_stat - statistics auto focus data + * + * @window: AF measured value of window x + * + * The module measures the sharpness in 3 windows of selectable size via + * register settings(ISP_AFM_*_A/B/C) + */ +struct cifisp_af_stat { + struct cifisp_af_meas_val window[CIFISP_AFM_MAX_WINDOWS]; +} __attribute__ ((packed)); + +/** + * struct cifisp_hist_stat - statistics histogram data + * + * @hist_bins: measured bin counters + * + * Measurement window divided into 25 sub-windows, set + * with ISP_HIST_XXX + */ +struct cifisp_hist_stat { + __u16 hist_bins[CIFISP_HIST_BIN_N_MAX]; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Data + * + * @cifisp_awb_stat: statistics data for automatic white balance + * @cifisp_ae_stat: statistics data for auto exposure + * @cifisp_af_stat: statistics data for auto focus + * @cifisp_hist_stat: statistics histogram data + */ +struct cifisp_stat { + struct cifisp_awb_stat awb; + struct cifisp_ae_stat ae; + struct cifisp_af_stat af; + struct cifisp_hist_stat hist; +} __attribute__ ((packed)); + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Meta Data + * + * @meas_type: measurement types (CIFISP_STAT_ definitions) + * @frame_id: frame ID for sync + * @params: statistics data + */ +struct rkisp1_stat_buffer { + __u32 meas_type; + __u32 frame_id; + struct cifisp_stat params; +} __attribute__ ((packed)); + +#endif /* _RKISP1_CONFIG_H */ diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index 5e739270116d3a21..94304bfeb4b6b9ed 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -727,6 +727,10 @@ struct v4l2_pix_format { #define V4L2_META_FMT_UVC v4l2_fourcc('U', 'V', 'C', 'H') /* UVC Payload Header metadata */ #define V4L2_META_FMT_D4XX v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */ +/* Vendor specific - used for RK_ISP1 camera sub-system */ +#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P') /* Rockchip ISP1 params */ +#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A statistics */ + /* priv field value to indicates that subsequent fields are valid. */ #define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe From patchwork Fri Sep 27 02:44:16 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2035 Return-Path: Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CB0386191D for ; Fri, 27 Sep 2019 04:45:35 +0200 (CEST) X-Halon-ID: d2599c4a-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id d2599c4a-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:10 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:16 +0200 Message-Id: <20190927024417.725906-13-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 12/13] libcamera: ipa: rkisp1: Add basic control of auto exposure X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:36 -0000 Add an IPA which controls the exposure time and analog gain for a sensor connected to the rkisp1 pipeline. The IPA supports turning AE on and off and informing the camera of the status of the AE control loop. Signed-off-by: Niklas Söderlund --- src/ipa/ipa_rkisp1.cpp | 228 +++++++++++++++++++++++++++++++++++++++++ src/ipa/meson.build | 13 +++ 2 files changed, 241 insertions(+) create mode 100644 src/ipa/ipa_rkisp1.cpp diff --git a/src/ipa/ipa_rkisp1.cpp b/src/ipa/ipa_rkisp1.cpp new file mode 100644 index 0000000000000000..f90465516c6aff87 --- /dev/null +++ b/src/ipa/ipa_rkisp1.cpp @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * ipa_rkisp1.cpp - RkISP1 Image Processing Algorithms + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "log.h" +#include "utils.h" + +#define BUFFER_PARAM 1 +#define BUFFER_STAT 2 + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPARkISP1) + +class IPARkISP1 : public IPAInterface +{ +public: + int init() override { return 0; } + + void initSensor(const V4L2ControlInfoMap &controls) override; + void initBuffers(unsigned int type, + const std::vector &buffers) override; + void signalBuffer(unsigned int type, unsigned int id) override; + void queueRequest(unsigned int frame, const ControlList &controls) override; + +private: + void setControls(unsigned int frame); + void updateStatistics(unsigned int frame, BufferMemory &statistics); + + std::map> bufferInfo_; + std::map> bufferFrame_; + std::map> bufferFree_; + + /* Camera sensor controls. */ + bool autoExposure_; + uint64_t exposure_; + uint64_t minExposure_; + uint64_t maxExposure_; + uint64_t gain_; + uint64_t minGain_; + uint64_t maxGain_; +}; + +void IPARkISP1::initSensor(const V4L2ControlInfoMap &controls) +{ + const auto itExp = controls.find(V4L2_CID_EXPOSURE); + if (itExp == controls.end()) { + LOG(IPARkISP1, Error) << "Can't find exposure control"; + return; + } + + const auto itGain = controls.find(V4L2_CID_ANALOGUE_GAIN); + if (itGain == controls.end()) { + LOG(IPARkISP1, Error) << "Can't find gain control"; + return; + } + + autoExposure_ = true; + + minExposure_ = std::max(itExp->second.min(), 1); + maxExposure_ = itExp->second.max(); + exposure_ = minExposure_; + + minGain_ = std::max(itGain->second.min(), 1); + maxGain_ = itGain->second.max(); + gain_ = minGain_; + + LOG(IPARkISP1, Info) + << "Exposure: " << minExposure_ << "-" << maxExposure_ + << " Gain: " << minGain_ << "-" << maxGain_; + + setControls(0); +} + +void IPARkISP1::initBuffers(unsigned int type, + const std::vector &buffers) +{ + bufferInfo_[type].clear(); + for (unsigned int i = 0; i < buffers.size(); i++) { + bufferInfo_[type][i] = buffers[i]; + bufferFree_[type].push(i); + } +} + +void IPARkISP1::signalBuffer(unsigned int type, unsigned int id) +{ + if (type == BUFFER_STAT) { + unsigned int frame = bufferFrame_[type][id]; + BufferMemory &mem = bufferInfo_[type][id]; + updateStatistics(frame, mem); + } + + bufferFree_[type].push(id); +} + +void IPARkISP1::queueRequest(unsigned int frame, const ControlList &controls) +{ + /* Find buffers. */ + if (bufferFree_[BUFFER_PARAM].empty()) { + LOG(IPARkISP1, Error) << "Param buffer underrun"; + return; + } + + if (bufferFree_[BUFFER_STAT].empty()) { + LOG(IPARkISP1, Error) << "Statistics buffer underrun"; + return; + } + + unsigned int paramid = bufferFree_[BUFFER_PARAM].front(); + bufferFree_[BUFFER_PARAM].pop(); + unsigned int statid = bufferFree_[BUFFER_STAT].front(); + bufferFree_[BUFFER_STAT].pop(); + + bufferFrame_[BUFFER_PARAM][paramid] = frame; + bufferFrame_[BUFFER_STAT][statid] = frame; + + /* Prepare parameters buffer. */ + BufferMemory &mem = bufferInfo_[BUFFER_PARAM][paramid]; + rkisp1_isp_params_cfg *params = + static_cast(mem.planes()[0].mem()); + + memset(params, 0, sizeof(*params)); + + /* Auto Exposure on/off. */ + if (controls.contains(AeEnable)) { + autoExposure_ = controls[AeEnable].getBool(); + if (autoExposure_) + params->module_ens = CIFISP_MODULE_AEC; + + params->module_en_update = CIFISP_MODULE_AEC; + } + + /* Queue buffers to pipeline. */ + queueBuffer.emit(frame, BUFFER_PARAM, paramid); + queueBuffer.emit(frame, BUFFER_STAT, statid); +} + +void IPARkISP1::setControls(unsigned int frame) +{ + V4L2ControlList ctrls; + ctrls.add(V4L2_CID_EXPOSURE); + ctrls.add(V4L2_CID_ANALOGUE_GAIN); + ctrls[V4L2_CID_EXPOSURE]->setValue(exposure_); + ctrls[V4L2_CID_ANALOGUE_GAIN]->setValue(gain_); + + updateSensor.emit(frame, ctrls); +} + +void IPARkISP1::updateStatistics(unsigned int frame, BufferMemory &statistics) +{ + const rkisp1_stat_buffer *stats = + static_cast(statistics.planes()[0].mem()); + const cifisp_stat *params = &stats->params; + IPAMetaData metaData = {}; + + if (stats->meas_type & CIFISP_STAT_AUTOEXP) { + const cifisp_ae_stat *ae = ¶ms->ae; + + const unsigned int target = 60; + + unsigned int value = 0; + unsigned int num = 0; + for (int i = 0; i < CIFISP_AE_MEAN_MAX; i++) { + if (ae->exp_mean[i] > 15) { + value += ae->exp_mean[i]; + num++; + } + } + value /= num; + + double factor = (double)target / value; + + if (frame % 3 == 0) { + double tmp; + + tmp = factor * exposure_ * gain_ / minGain_; + exposure_ = utils::clamp((uint64_t)tmp, minExposure_, maxExposure_); + + tmp = tmp / exposure_ * minGain_; + gain_ = utils::clamp((uint64_t)tmp, minGain_, maxGain_); + + setControls(frame + 1); + } + + metaData.aeState = fabs(factor - 1.0f) < 0.05f ? + AeState::Converged : AeState::Searching; + } else { + metaData.aeState = AeState::Inactive; + } + + metaDataReady.emit(frame, metaData); +} + +/* + * External IPA module interface + */ + +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 1, + "PipelineHandlerRkISP1", + "RkISP1 IPA", + "LGPL-2.1-or-later", +}; + +IPAInterface *ipaCreate() +{ + return new IPARkISP1(); +} +}; + +}; /* namespace libcamera */ diff --git a/src/ipa/meson.build b/src/ipa/meson.build index 10448c2ffc76af4b..eb5b852eec282735 100644 --- a/src/ipa/meson.build +++ b/src/ipa/meson.build @@ -3,6 +3,10 @@ ipa_dummy_sources = [ ['ipa_dummy_isolate', 'Proprietary'], ] +ipa_sources = [ + ['ipa_rkisp1', 'ipa_rkisp1.cpp', 'LGPL-2.1-or-later'], +] + ipa_install_dir = join_paths(get_option('libdir'), 'libcamera') foreach t : ipa_dummy_sources @@ -14,5 +18,14 @@ foreach t : ipa_dummy_sources cpp_args : '-DLICENSE="' + t[1] + '"') endforeach +foreach t : ipa_sources + ipa = shared_module(t[0], t[1], + name_prefix : '', + include_directories : includes, + install : true, + install_dir : ipa_install_dir, + cpp_args : '-DLICENSE="' + t[2] + '"') +endforeach + config_h.set('IPA_MODULE_DIR', '"' + join_paths(get_option('prefix'), ipa_install_dir) + '"') From patchwork Fri Sep 27 02:44:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2037 Return-Path: Received: from bin-mail-out-05.binero.net (bin-mail-out-05.binero.net [195.74.38.228]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id E25F56191D for ; Fri, 27 Sep 2019 04:45:37 +0200 (CEST) X-Halon-ID: d2daf8dd-e0d0-11e9-bdc3-005056917a89 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-01.atm.binero.net (Halon) with ESMTPA id d2daf8dd-e0d0-11e9-bdc3-005056917a89; Fri, 27 Sep 2019 04:45:10 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Sep 2019 04:44:17 +0200 Message-Id: <20190927024417.725906-14-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> References: <20190927024417.725906-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 13/13] libcamera: pipeline: rkisp1: Attach to an IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 27 Sep 2019 02:45:38 -0000 Add the plumbing to the pipeline handler to interact with an IPA module. This change makes the usage of an IPA module mandatory for the rkisp1 pipeline. Signed-off-by: Niklas Söderlund --- src/libcamera/pipeline/rkisp1/meson.build | 1 + src/libcamera/pipeline/rkisp1/rkisp1.cpp | 263 ++++++++++++++++++++- src/libcamera/pipeline/rkisp1/rkisp1.h | 81 +++++++ src/libcamera/pipeline/rkisp1/timeline.cpp | 56 +++++ 4 files changed, 388 insertions(+), 13 deletions(-) create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1.h create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build index f1cc4046b5d064cb..d04fb45223e72fa1 100644 --- a/src/libcamera/pipeline/rkisp1/meson.build +++ b/src/libcamera/pipeline/rkisp1/meson.build @@ -1,3 +1,4 @@ libcamera_sources += files([ 'rkisp1.cpp', + 'timeline.cpp', ]) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index de4ab523d0e4fe36..5ca2c93bc7e70100 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -5,11 +5,12 @@ * rkisp1.cpp - Pipeline handler for Rockchip ISP1 */ +#include "rkisp1.h" + #include #include #include #include -#include #include @@ -17,8 +18,8 @@ #include #include -#include "camera_sensor.h" #include "device_enumerator.h" +#include "ipa_manager.h" #include "log.h" #include "media_device.h" #include "pipeline_handler.h" @@ -34,7 +35,7 @@ class RkISP1CameraData : public CameraData { public: RkISP1CameraData(PipelineHandler *pipe) - : CameraData(pipe), sensor_(nullptr) + : CameraData(pipe), sensor_(nullptr), frame_(0) { } @@ -43,8 +44,19 @@ public: delete sensor_; } + int loadIPA() override; + Stream stream_; CameraSensor *sensor_; + unsigned int frame_; + std::map frameInfo_; + RkISP1Timeline timeline_; + +private: + void updateSensor(unsigned int frame, V4L2ControlList controls); + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metaDataReady(unsigned int frame, IPAMetaData metaData); }; class RkISP1CameraConfiguration : public CameraConfiguration @@ -99,18 +111,94 @@ private: PipelineHandler::cameraData(camera)); } + friend RkISP1CameraData; + int initLinks(); int createCamera(MediaEntity *sensor); + void tryCompleteRequest(Request *request); void bufferReady(Buffer *buffer); + void paramReady(Buffer *buffer); + void statReady(Buffer *buffer); MediaDevice *media_; V4L2Subdevice *dphy_; V4L2Subdevice *isp_; V4L2VideoDevice *video_; + V4L2VideoDevice *param_; + V4L2VideoDevice *stat_; + + BufferPool paramPool_; + BufferPool statPool_; + + std::map paramBuffers_; + std::map statBuffers_; Camera *activeCamera_; }; +int RkISP1CameraData::loadIPA() +{ + ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->updateSensor.connect(this, + &RkISP1CameraData::updateSensor); + ipa_->queueBuffer.connect(this, + &RkISP1CameraData::queueBuffer); + ipa_->setDelay.connect(&timeline_, + &RkISP1Timeline::setDelay); + ipa_->metaDataReady.connect(this, + &RkISP1CameraData::metaDataReady); + + return 0; +} + +void RkISP1CameraData::updateSensor(unsigned int frame, V4L2ControlList controls) +{ + timeline_.scheduleAction(new RkISP1ActionSetSensor(frame, sensor_, controls)); +} + +void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + RkISP1ActionType acttype; + V4L2VideoDevice *device; + Buffer *buffer; + switch (type) { + case BUFFER_PARAM: + acttype = QueueParameters; + device = pipe->param_; + buffer = pipe->paramBuffers_[id]; + break; + case BUFFER_STAT: + acttype = QueueStatistics; + device = pipe->stat_; + buffer = pipe->statBuffers_[id]; + break; + default: + LOG(RkISP1, Error) << "Unkown IPA buffer type " << type; + return; + } + + timeline_.scheduleAction(new RkISP1ActionQueueBuffer(frame, acttype, + device, buffer)); +} + +void RkISP1CameraData::metaDataReady(unsigned int frame, IPAMetaData metaData) +{ + Request *request = frameInfo_[frame]; + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + pipe->processMetaData(request, metaData); + + pipe->tryCompleteRequest(request); +} + RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera, RkISP1CameraData *data) : CameraConfiguration() @@ -202,12 +290,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) : PipelineHandler(manager), dphy_(nullptr), isp_(nullptr), - video_(nullptr) + video_(nullptr), param_(nullptr), stat_(nullptr) { } PipelineHandlerRkISP1::~PipelineHandlerRkISP1() { + delete param_; + delete stat_; delete video_; delete isp_; delete dphy_; @@ -317,6 +407,20 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) if (ret) return ret; + V4L2DeviceFormat paramFormat = {}; + paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS; + + ret = param_->setFormat(¶mFormat); + if (ret) + return ret; + + V4L2DeviceFormat statFormat = {}; + statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A; + + ret = stat_->setFormat(&statFormat); + if (ret) + return ret; + if (outputFormat.size != cfg.size || outputFormat.fourcc != cfg.pixelFormat) { LOG(RkISP1, Error) @@ -332,31 +436,99 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) int PipelineHandlerRkISP1::allocateBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); Stream *stream = *streams.begin(); + int ret; if (stream->memoryType() == InternalMemory) - return video_->exportBuffers(&stream->bufferPool()); + ret = video_->exportBuffers(&stream->bufferPool()); else - return video_->importBuffers(&stream->bufferPool()); + ret = video_->importBuffers(&stream->bufferPool()); + + if (ret) + return ret; + + paramPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = param_->exportBuffers(¶mPool_); + if (ret) { + video_->releaseBuffers(); + return ret; + } + + statPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = stat_->exportBuffers(&statPool_); + if (ret) { + param_->releaseBuffers(); + video_->releaseBuffers(); + return ret; + } + + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) { + paramBuffers_[i] = new Buffer(i); + statBuffers_[i] = new Buffer(i); + } + + data->ipa_->initBuffers(BUFFER_PARAM, paramPool_.buffers()); + data->ipa_->initBuffers(BUFFER_STAT, statPool_.buffers()); + + return ret; } int PipelineHandlerRkISP1::freeBuffers(Camera *camera, const std::set &streams) { + for (auto it : paramBuffers_) + delete it.second; + + paramBuffers_.clear(); + + for (auto it : statBuffers_) + delete it.second; + + statBuffers_.clear(); + + if (param_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release parameters buffers"; + + if (stat_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release stat buffers"; + if (video_->releaseBuffers()) - LOG(RkISP1, Error) << "Failed to release buffers"; + LOG(RkISP1, Error) << "Failed to release video buffers"; return 0; } int PipelineHandlerRkISP1::start(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; + data->ipa_->initSensor(data->sensor_->controls()); + + ret = param_->streamOn(); + if (ret) { + LOG(RkISP1, Error) + << "Failed to start parameters " << camera->name(); + return ret; + } + + ret = stat_->streamOn(); + if (ret) { + param_->streamOff(); + LOG(RkISP1, Error) + << "Failed to start statistics " << camera->name(); + return ret; + } + ret = video_->streamOn(); - if (ret) + if (ret) { + param_->streamOff(); + stat_->streamOff(); + LOG(RkISP1, Error) << "Failed to start camera " << camera->name(); + } activeCamera_ = camera; @@ -365,6 +537,7 @@ int PipelineHandlerRkISP1::start(Camera *camera) void PipelineHandlerRkISP1::stop(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; ret = video_->streamOff(); @@ -372,6 +545,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera) LOG(RkISP1, Warning) << "Failed to stop camera " << camera->name(); + ret = stat_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop statistics " << camera->name(); + + ret = param_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop parameters " << camera->name(); + + data->timeline_.reset(); + activeCamera_ = nullptr; } @@ -387,12 +572,17 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request) return -ENOENT; } - int ret = video_->queueBuffer(buffer); - if (ret < 0) - return ret; - PipelineHandler::queueRequest(camera, request); + data->frameInfo_[data->frame_] = request; + data->ipa_->queueRequest(data->frame_, request->controls()); + + data->timeline_.scheduleAction(new RkISP1ActionQueueBuffer(data->frame_, + QueueVideo, + video_, + buffer)); + data->frame_++; + return 0; } @@ -435,6 +625,10 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) std::unique_ptr data = utils::make_unique(this); + data->controlInfo_.emplace(std::piecewise_construct, + std::forward_as_tuple(AeEnable), + std::forward_as_tuple(AeEnable, false, true)); + data->sensor_ = new CameraSensor(sensor); ret = data->sensor_->init(); if (ret) @@ -478,7 +672,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) if (video_->open() < 0) return false; + stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics"); + if (stat_->open() < 0) + return false; + + param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params"); + if (param_->open() < 0) + return false; + video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady); + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); /* Configure default links. */ if (initLinks() < 0) { @@ -504,13 +708,46 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) * Buffer Handling */ +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request) +{ + if (request->hasPendingBuffers()) + return; + + if (!request->metaData().ready) + return; + + completeRequest(activeCamera_, request); +} + void PipelineHandlerRkISP1::bufferReady(Buffer *buffer) { ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); Request *request = buffer->request(); + data->timeline_.bufferReady(buffer); + + if (data->frame_ <= buffer->sequence()) + data->frame_ = buffer->sequence() + 1; + completeBuffer(activeCamera_, request, buffer); - completeRequest(activeCamera_, request); + tryCompleteRequest(request); +} + +void PipelineHandlerRkISP1::paramReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + data->ipa_->signalBuffer(BUFFER_PARAM, buffer->index()); +} + +void PipelineHandlerRkISP1::statReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + data->ipa_->signalBuffer(BUFFER_STAT, buffer->index()); } REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1); diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h new file mode 100644 index 0000000000000000..aea0eaa6bc838755 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/rkisp1.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Pipeline handler for Rockchip ISP1 + */ +#ifndef __LIBCAMERA_RKISP1_H__ +#define __LIBCAMERA_RKISP1_H__ + +#include "timeline.h" + +#include + +#include "camera_sensor.h" +#include "v4l2_videodevice.h" + +#define BUFFER_PARAM 1 +#define BUFFER_STAT 2 + +namespace libcamera { + +enum RkISP1ActionType { + SetSensor, + SOE, + QueueVideo, + QueueParameters, + QueueStatistics, +}; + +class RkISP1ActionSetSensor : public FrameAction +{ +public: + RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls) + : FrameAction(SetSensor, frame), sensor_(sensor), controls_(controls) {} + +protected: + void run() override; + +private: + CameraSensor *sensor_; + V4L2ControlList controls_; +}; + +class RkISP1ActionQueueBuffer : public FrameAction +{ +public: + RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type, + V4L2VideoDevice *device, Buffer *buffer) + : FrameAction(type, frame), device_(device), buffer_(buffer) + { + } + +protected: + void run() override; + +private: + V4L2VideoDevice *device_; + Buffer *buffer_; +}; + +class RkISP1Timeline : public Timeline +{ +public: + RkISP1Timeline() + : Timeline() + { + setDelay(SetSensor, -1, 5); + setDelay(SOE, 0, -1); + setDelay(QueueVideo, -1, 10); + setDelay(QueueParameters, -1, 8); + setDelay(QueueStatistics, -1, 8); + } + + void bufferReady(Buffer *buffer); + + void setDelay(unsigned int type, int frame, int msdelay); +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_RKISP1_H__ */ diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp new file mode 100644 index 0000000000000000..ca0cd96711d84244 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline handler for Rockchip ISP1 + */ + +#include "rkisp1.h" + +#include "log.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(RkISP1) + +void RkISP1ActionSetSensor::run() +{ + sensor_->setControls(&controls_); +} + +void RkISP1ActionQueueBuffer::run() +{ + int ret = device_->queueBuffer(buffer_); + if (ret < 0) + LOG(RkISP1, Error) << "Failed to queue buffer"; +} + +void RkISP1Timeline::bufferReady(Buffer *buffer) +{ + /* + * Calculate SOE by taking the end of DMA set by the kernel and applying + * the time offsets provieded by the IPA to find the best estimate of + * SOE. + * + * NOTE: Make sure the IPA do not set a frame offset for the SOE action + * type as the frame interval is calculated using the interval between + * two SOE events. So using a frame interval in the SOE estimate creates + * a recursion. + */ + + ASSERT(frameOffset(SOE) == 0); + + utils::time_point soe = std::chrono::time_point() + + std::chrono::nanoseconds(buffer->timestamp()) + + timeOffset(SOE); + + notifyStartOfExposure(buffer->sequence(), soe); +} + +void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay) +{ + utils::duration delay = std::chrono::milliseconds(msdelay); + setRawDelay(type, frame, delay); +} + +} /* namespace libcamera */