From patchwork Fri Jul 26 11:47:11 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 20710 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 3B0BDBDC71 for ; Fri, 26 Jul 2024 11:47:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C6D2C63373; Fri, 26 Jul 2024 13:47:31 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Gl9HeMPr"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 12E6163369 for ; Fri, 26 Jul 2024 13:47:28 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B3D1383F; Fri, 26 Jul 2024 13:46:42 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1721994403; bh=rc0UIFtaBBkpr1PAft53WyM0kK/fASh5evObxVEuVMA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Gl9HeMPrzUnDXlGXPlI7QwbliUioy8DSqjgru3yyF6qw+WSFFatUgoKV2DV6AYoOZ psByjmZm+00j0/ZTpRMWfNJz341b3CYV+ka5XFVOgVMW4+ZLV/J9Dn68NsFrxYi5np J7j/mrRK/Xey70d6WS85sx2Wy4rXl8nQZ3onJnXo= From: Umang Jain To: libcamera-devel@lists.libcamera.org Cc: Umang Jain , Kieran Bingham , Paul Elder Subject: [PATCH v6 1/5] libcamera: converter: Add interface for feature flags Date: Fri, 26 Jul 2024 17:17:11 +0530 Message-ID: <20240726114715.226468-2-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.45.0 In-Reply-To: <20240726114715.226468-1-umang.jain@ideasonboard.com> References: <20240726114715.226468-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" This patch intends to extend the converter interface to have feature flags, which enables each converter to expose the set of features it supports. Signed-off-by: Umang Jain Reviewed-by: Kieran Bingham Reviewed-by: Paul Elder Reviewed-by: Laurent Pinchart --- include/libcamera/internal/converter.h | 13 ++++++++++- .../internal/converter/converter_v4l2_m2m.h | 2 +- src/libcamera/converter.cpp | 22 ++++++++++++++++++- .../converter/converter_v4l2_m2m.cpp | 5 +++-- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/include/libcamera/internal/converter.h b/include/libcamera/internal/converter.h index b51563d7..652ff519 100644 --- a/include/libcamera/internal/converter.h +++ b/include/libcamera/internal/converter.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -32,7 +33,13 @@ struct StreamConfiguration; class Converter { public: - Converter(MediaDevice *media); + enum class Feature { + None = 0, + }; + + using Features = Flags; + + Converter(MediaDevice *media, Features features = Feature::None); virtual ~Converter(); virtual int loadConfiguration(const std::string &filename) = 0; @@ -61,8 +68,12 @@ public: const std::string &deviceNode() const { return deviceNode_; } + Features features() const { return features_; } + private: std::string deviceNode_; + + Features features_; }; class ConverterFactoryBase diff --git a/include/libcamera/internal/converter/converter_v4l2_m2m.h b/include/libcamera/internal/converter/converter_v4l2_m2m.h index b9e59899..91701dbe 100644 --- a/include/libcamera/internal/converter/converter_v4l2_m2m.h +++ b/include/libcamera/internal/converter/converter_v4l2_m2m.h @@ -35,7 +35,7 @@ class V4L2M2MDevice; class V4L2M2MConverter : public Converter { public: - V4L2M2MConverter(MediaDevice *media); + V4L2M2MConverter(MediaDevice *media, Features features = Feature::None); int loadConfiguration([[maybe_unused]] const std::string &filename) { return 0; } bool isValid() const { return m2m_ != nullptr; } diff --git a/src/libcamera/converter.cpp b/src/libcamera/converter.cpp index 2ab46133..e2dbf5e0 100644 --- a/src/libcamera/converter.cpp +++ b/src/libcamera/converter.cpp @@ -34,14 +34,27 @@ LOG_DEFINE_CATEGORY(Converter) * parameters from the same input stream. */ +/** + * \enum Converter::Feature + * \brief Specify the features supported by the converter + * \var Converter::Feature::None + * \brief No extra features supported by the converter + */ + +/** + * \typedef Converter::Features + * \brief A bitwise combination of features supported by the converter + */ + /** * \brief Construct a Converter instance * \param[in] media The media device implementing the converter + * \param[in] features Features flags representing supported features * * This searches for the entity implementing the data streaming function in the * media graph entities and use its device node as the converter device node. */ -Converter::Converter(MediaDevice *media) +Converter::Converter(MediaDevice *media, Features features) { const std::vector &entities = media->entities(); auto it = std::find_if(entities.begin(), entities.end(), @@ -56,6 +69,7 @@ Converter::Converter(MediaDevice *media) } deviceNode_ = (*it)->deviceNode(); + features_ = features; } Converter::~Converter() @@ -163,6 +177,12 @@ Converter::~Converter() * \return The converter device node string */ +/** + * \fn Converter::features() + * \brief Gets the supported features by the converter + * \return The converter Features flags + */ + /** * \class ConverterFactoryBase * \brief Base class for converter factories diff --git a/src/libcamera/converter/converter_v4l2_m2m.cpp b/src/libcamera/converter/converter_v4l2_m2m.cpp index 2e77872e..4aeb7dd9 100644 --- a/src/libcamera/converter/converter_v4l2_m2m.cpp +++ b/src/libcamera/converter/converter_v4l2_m2m.cpp @@ -191,10 +191,11 @@ void V4L2M2MConverter::V4L2M2MStream::captureBufferReady(FrameBuffer *buffer) * \fn V4L2M2MConverter::V4L2M2MConverter * \brief Construct a V4L2M2MConverter instance * \param[in] media The media device implementing the converter + * \param[in] features Features flags representing supported features */ -V4L2M2MConverter::V4L2M2MConverter(MediaDevice *media) - : Converter(media) +V4L2M2MConverter::V4L2M2MConverter(MediaDevice *media, Features features) + : Converter(media, features) { if (deviceNode().empty()) return; From patchwork Fri Jul 26 11:47:12 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 20711 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id AEB9CBDC71 for ; Fri, 26 Jul 2024 11:47:34 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4E0A963378; Fri, 26 Jul 2024 13:47:34 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="jyTvf4/T"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 8E8466336B for ; Fri, 26 Jul 2024 13:47:29 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 5D5378CC; Fri, 26 Jul 2024 13:46:44 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1721994405; bh=l+JMY3xVM7qlcAJq3/PvQ9triNKZiGSv4dbVblgMnQ0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=jyTvf4/TIWIRRFndX3LgFENhhwhcyL5qqa48vLmQMuZGdykob0GOFGHB0HT5W6IuH ebVS9Rg1xPhxRnmbWktOnVaADCt0FPYiMmf1Z5FN1UQtSzuIghI2yE7om6UCjWSoNH bQZHNHYLAptYUUu62p9+h6hdGtPOn2gD6+2pi1AE= From: Umang Jain To: libcamera-devel@lists.libcamera.org Cc: Umang Jain , Paul Elder Subject: [PATCH v6 2/5] libcamera: converter: Add interface to support cropping capability Date: Fri, 26 Jul 2024 17:17:12 +0530 Message-ID: <20240726114715.226468-3-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.45.0 In-Reply-To: <20240726114715.226468-1-umang.jain@ideasonboard.com> References: <20240726114715.226468-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" If the converter has cropping capability, the interface should support it by providing appropriate virtual functions. Provide Feature::Crop in Feature enumeration for the same. Provide virtual setCrop() and getCropBounds() interfaces so that the converter can implement their own cropping functionality. The V4L2M2MConverter implements these interfaces of the Converter interface. Not all V4L2M2M converters will have cropping capability, hence Feature::Crop should be set while construction for all those converters. Signed-off-by: Umang Jain Reviewed-by: Paul Elder --- include/libcamera/internal/converter.h | 5 + .../internal/converter/converter_v4l2_m2m.h | 6 ++ src/libcamera/converter.cpp | 52 +++++++++++ .../converter/converter_v4l2_m2m.cpp | 92 +++++++++++++++++++ 4 files changed, 155 insertions(+) diff --git a/include/libcamera/internal/converter.h b/include/libcamera/internal/converter.h index 652ff519..853888f4 100644 --- a/include/libcamera/internal/converter.h +++ b/include/libcamera/internal/converter.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,7 @@ class Converter public: enum class Feature { None = 0, + InputCrop = (1 << 0), }; using Features = Flags; @@ -63,6 +65,9 @@ public: virtual int queueBuffers(FrameBuffer *input, const std::map &outputs) = 0; + virtual int setInputCrop(const Stream *stream, Rectangle *rect); + virtual std::pair inputCropBounds(const Stream *stream); + Signal inputBufferReady; Signal outputBufferReady; diff --git a/include/libcamera/internal/converter/converter_v4l2_m2m.h b/include/libcamera/internal/converter/converter_v4l2_m2m.h index 91701dbe..5b69b14e 100644 --- a/include/libcamera/internal/converter/converter_v4l2_m2m.h +++ b/include/libcamera/internal/converter/converter_v4l2_m2m.h @@ -30,6 +30,7 @@ class Size; class SizeRange; class Stream; struct StreamConfiguration; +class Rectangle; class V4L2M2MDevice; class V4L2M2MConverter : public Converter @@ -57,6 +58,9 @@ public: int queueBuffers(FrameBuffer *input, const std::map &outputs); + int setInputCrop(const Stream *stream, Rectangle *rect); + std::pair inputCropBounds(const Stream *stream); + private: class V4L2M2MStream : protected Loggable { @@ -75,6 +79,8 @@ private: int queueBuffers(FrameBuffer *input, FrameBuffer *output); + int setSelection(unsigned int target, Rectangle *rect); + protected: std::string logPrefix() const override; diff --git a/src/libcamera/converter.cpp b/src/libcamera/converter.cpp index e2dbf5e0..7ab56b4c 100644 --- a/src/libcamera/converter.cpp +++ b/src/libcamera/converter.cpp @@ -11,6 +11,8 @@ #include +#include + #include "libcamera/internal/media_device.h" /** @@ -39,6 +41,8 @@ LOG_DEFINE_CATEGORY(Converter) * \brief Specify the features supported by the converter * \var Converter::Feature::None * \brief No extra features supported by the converter + * \var Converter::Feature::InputCrop + * \brief Cropping capability at input is supported by the converter */ /** @@ -161,6 +165,54 @@ Converter::~Converter() * \return 0 on success or a negative error code otherwise */ +/** + * \brief Set the crop rectangle \a rect for \a stream + * \param[in] stream Pointer to output stream + * \param[inout] rect The crop rectangle to be applied + * + * Set the crop rectangle \a rect for \a stream provided the converter supports + * cropping. The converter should have the Feature::InputCrop flag in this + * case. + * + * \return 0 on success or a negative error code otherwise + */ +int Converter::setInputCrop([[maybe_unused]] const Stream *stream, [[maybe_unused]] Rectangle *rect) +{ + if (!(features() & Feature::InputCrop)) { + LOG(Converter, Error) << "Converter doesn't support cropping capabilities"; + return -ENOTSUP; + } + + return 0; +} + +/** + * \brief Get the crop bounds \a stream + * \param[in] stream Pointer to output stream + * + * Get the minimum and maximum crop bounds for \a stream. The converter + * should supporting cropping (Feature::InputCrop). + * + * \return A std::pair containining minimum and maximum + * crop bound respectively. + */ +std::pair Converter::inputCropBounds([[maybe_unused]] const Stream *stream) +{ + const StreamConfiguration &config = stream->configuration(); + Rectangle rect; + + if (!(features() & Feature::InputCrop)) + LOG(Converter, Error) << "Converter doesn't support cropping capabilities"; + + /* + * This is base implementation for the Converter class, so just return + * the stream configured size as minimum and maximum crop bounds. + */ + rect.size() = config.size; + + return { rect, rect }; +} + /** * \var Converter::inputBufferReady * \brief A signal emitted when the input frame buffer completes diff --git a/src/libcamera/converter/converter_v4l2_m2m.cpp b/src/libcamera/converter/converter_v4l2_m2m.cpp index 4aeb7dd9..3295b988 100644 --- a/src/libcamera/converter/converter_v4l2_m2m.cpp +++ b/src/libcamera/converter/converter_v4l2_m2m.cpp @@ -155,6 +155,15 @@ int V4L2M2MConverter::V4L2M2MStream::queueBuffers(FrameBuffer *input, FrameBuffe return 0; } +int V4L2M2MConverter::V4L2M2MStream::setSelection(unsigned int target, Rectangle *rect) +{ + int ret = m2m_->output()->setSelection(target, rect); + if (ret < 0) + return ret; + + return 0; +} + std::string V4L2M2MConverter::V4L2M2MStream::logPrefix() const { return stream_->configuration().toString(); @@ -375,6 +384,85 @@ int V4L2M2MConverter::exportBuffers(const Stream *stream, unsigned int count, return iter->second->exportBuffers(count, buffers); } +/** + * \copydoc libcamera::Converter::setInputCrop + */ +int V4L2M2MConverter::setInputCrop(const Stream *stream, Rectangle *rect) +{ + if (!(features() & Feature::InputCrop)) + return -ENOTSUP; + + auto iter = streams_.find(stream); + if (iter == streams_.end()) + return -EINVAL; + + return iter->second->setSelection(V4L2_SEL_TGT_CROP, rect); +} + +/** + * \copydoc libcamera::Converter::inputCropBounds + */ +std::pair +V4L2M2MConverter::inputCropBounds(const Stream *stream) +{ + Rectangle minCrop; + Rectangle maxCrop; + int ret; + + if (!(features() & Feature::InputCrop)) { + LOG(Converter, Error) << "Input Crop functionality is not supported"; + return {}; + } + + minCrop.width = 1; + minCrop.height = 1; + maxCrop.width = UINT_MAX; + maxCrop.height = UINT_MAX; + + auto iter = streams_.find(stream); + if (iter == streams_.end()) { + /* + * No streams configured, return minimum and maximum crop + * bounds at initialization. + */ + ret = m2m_->output()->setSelection(V4L2_SEL_TGT_CROP, &minCrop); + if (ret) { + LOG(Converter, Error) << "Failed to query minimum crop bound" + << strerror(-ret); + return {}; + } + + ret = m2m_->output()->setSelection(V4L2_SEL_TGT_CROP, &maxCrop); + if (ret) { + LOG(Converter, Error) << "Failed to query maximum crop bound" + << strerror(-ret); + return {}; + } + + return { minCrop, maxCrop }; + } + + /* + * If the streams are configured, return bounds from according to + * stream configuration. + */ + ret = setInputCrop(stream, &minCrop); + if (ret) { + LOG(Converter, Error) << "Failed to query minimum crop bound" + << strerror(-ret); + return {}; + } + + ret = setInputCrop(stream, &maxCrop); + if (ret) { + LOG(Converter, Error) << "Failed to query maximum crop bound" + << strerror(-ret); + return {}; + } + + return { minCrop, maxCrop }; +} + /** * \copydoc libcamera::Converter::start */ @@ -448,6 +536,10 @@ int V4L2M2MConverter::queueBuffers(FrameBuffer *input, return 0; } +/* + * \todo: This should be extended to include Feature::Flag to denote + * what each converter supports feature-wise. + */ static std::initializer_list compatibles = { "mtk-mdp", "pxp", From patchwork Fri Jul 26 11:47:13 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 20712 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 1A276BDC71 for ; Fri, 26 Jul 2024 11:47:37 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 7FA436336F; Fri, 26 Jul 2024 13:47:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="dg3rEVHk"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 0F8856199C for ; Fri, 26 Jul 2024 13:47:31 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id CAE7783F; Fri, 26 Jul 2024 13:46:45 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1721994406; bh=zKwOk/QMMyTFPG9Frd0wSQWkxZJESCuUxSxZQARTcA0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=dg3rEVHkUFJ43efzLbIqgmVK/inb8MxtnV0cYJ71zuj/a0QlSWmyOGLm9Wh7X13Lg Ee96IzbyVzp54jyvjQLx6ny8Z4LZ9o5z8Xs5UvobgcTJi11+BEwRCrz72PA3fMBX2c zCAsNXbX9yfVkpmWrPuAawAOlfgO19dazEEqK8gc= From: Umang Jain To: libcamera-devel@lists.libcamera.org Cc: Umang Jain , Paul Elder Subject: [PATCH v6 3/5] libcamera: rkisp1: Prepare for additional camera controls Date: Fri, 26 Jul 2024 17:17:13 +0530 Message-ID: <20240726114715.226468-4-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.45.0 In-Reply-To: <20240726114715.226468-1-umang.jain@ideasonboard.com> References: <20240726114715.226468-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Currently the rkisp1 pipeline handler only registers controls that are related to the IPA. This patch prepares the rkisp1 pipeline-handler to register camera controls which are not related to the IPA. Hence, introduce a additional ControlInfoMap for IPA controls. These controls will be merged together with the controls in the pipeline handler (introduced subsequently) as part of updateControls() and together will be registered during the registration of the camera. This is similar to what IPU3 pipeline handler handles its controls. Signed-off-by: Umang Jain Reviewed-by: Paul Elder --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index 5f94f422..25f2cc97 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -109,6 +109,8 @@ public: std::unique_ptr ipa_; + ControlInfoMap ipaControls_; + private: void paramFilled(unsigned int frame); void setSensorControls(unsigned int frame, @@ -184,6 +186,8 @@ private: int allocateBuffers(Camera *camera); int freeBuffers(Camera *camera); + int updateControls(RkISP1CameraData *data); + MediaDevice *media_; std::unique_ptr isp_; std::unique_ptr param_; @@ -370,7 +374,7 @@ int RkISP1CameraData::loadIPA(unsigned int hwRevision) } ret = ipa_->init({ ipaTuningFile, sensor_->model() }, hwRevision, - sensorInfo, sensor_->controls(), &controlInfo_); + sensorInfo, sensor_->controls(), &ipaControls_); if (ret < 0) { LOG(RkISP1, Error) << "IPA initialization failure"; return ret; @@ -820,12 +824,13 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) ipaConfig.sensorControls = data->sensor_->controls(); - ret = data->ipa_->configure(ipaConfig, streamConfig, &data->controlInfo_); + ret = data->ipa_->configure(ipaConfig, streamConfig, &data->ipaControls_); if (ret) { LOG(RkISP1, Error) << "failed configuring IPA (" << ret << ")"; return ret; } - return 0; + + return updateControls(data); } int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, Stream *stream, @@ -1101,8 +1106,23 @@ int PipelineHandlerRkISP1::initLinks(Camera *camera, return 0; } +int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data) +{ + ControlInfoMap::Map rkisp1Controls; + + /* Add the IPA registered controls to list of camera controls. */ + for (const auto &ipaControl : data->ipaControls_) + rkisp1Controls[ipaControl.first] = ipaControl.second; + + data->controlInfo_ = ControlInfoMap(std::move(rkisp1Controls), + controls::controls); + + return 0; +} + int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) { + ControlInfoMap::Map rkisp1Controls; int ret; std::unique_ptr data = @@ -1137,6 +1157,8 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) if (ret) return ret; + updateControls(data.get()); + std::set streams{ &data->mainPathStream_, &data->selfPathStream_, From patchwork Fri Jul 26 11:47:14 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 20713 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C0A11BDC71 for ; Fri, 26 Jul 2024 11:47:39 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4DE6663378; Fri, 26 Jul 2024 13:47:38 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="uxeFkhNk"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 73D2A6199C for ; Fri, 26 Jul 2024 13:47:32 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 47ECA8CC; Fri, 26 Jul 2024 13:46:47 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1721994408; bh=g1L5ROJltTGOPcYCYXqk9+ly2EHbHenoJecEotA7u/o=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=uxeFkhNkE9V6H2kDpLNI0+2U6hVjz7B48RkDAR5UEOSrFbnEsfYpiW/jCs0bqF5/1 SxCoAAR5pEJqtOeYja/Xl4yhETwlWV2hD7Lg0oud/N2VKOHEJCqqyNF8Fg1sV9ngMw dOyHdNLj+i265jGJo2Msy+AkpmFZxT2MXBjH2+m0= From: Umang Jain To: libcamera-devel@lists.libcamera.org Cc: Umang Jain , Paul Elder Subject: [PATCH v6 4/5] libcamera: converter: Add DW100 converter class Date: Fri, 26 Jul 2024 17:17:14 +0530 Message-ID: <20240726114715.226468-5-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.45.0 In-Reply-To: <20240726114715.226468-1-umang.jain@ideasonboard.com> References: <20240726114715.226468-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" DW100 Dewarp engine is present on i.MX8MP SoC. This patch provides a converter class (inherited from V4L2M2MConverter). The DW100 dewarp engine has scaling capabilites and can be used to set crop rectangles in the rkisp1 pipeline handler. Plumbing in the RkISP1 pipeline-handler is done in the subsequent patch. The ConverterDW100 class can be extended later (as additional development) to support the dewarping by setting a dewarp vertex map as well. Signed-off-by: Umang Jain Reviewed-by: Paul Elder --- .../internal/converter/converter_dw100.h | 24 ++++++++++++ .../libcamera/internal/converter/meson.build | 1 + src/libcamera/converter/converter_dw100.cpp | 37 +++++++++++++++++++ src/libcamera/converter/meson.build | 1 + 4 files changed, 63 insertions(+) create mode 100644 include/libcamera/internal/converter/converter_dw100.h create mode 100644 src/libcamera/converter/converter_dw100.cpp diff --git a/include/libcamera/internal/converter/converter_dw100.h b/include/libcamera/internal/converter/converter_dw100.h new file mode 100644 index 00000000..dc41f365 --- /dev/null +++ b/include/libcamera/internal/converter/converter_dw100.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * i.MX8MP Dewarp Engine integration + */ + +#pragma once + +#include "libcamera/internal/converter/converter_v4l2_m2m.h" + +namespace libcamera { + +class MediaDevice; +class Rectangle; +class Stream; + +class ConverterDW100 : public V4L2M2MConverter +{ +public: + ConverterDW100(std::shared_ptr media); +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/converter/meson.build b/include/libcamera/internal/converter/meson.build index 891e79e7..85007a4b 100644 --- a/include/libcamera/internal/converter/meson.build +++ b/include/libcamera/internal/converter/meson.build @@ -1,5 +1,6 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_headers += files([ + 'converter_dw100.h', 'converter_v4l2_m2m.h', ]) diff --git a/src/libcamera/converter/converter_dw100.cpp b/src/libcamera/converter/converter_dw100.cpp new file mode 100644 index 00000000..eb9906f0 --- /dev/null +++ b/src/libcamera/converter/converter_dw100.cpp @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * i.MX8MP Dewarp Engine integration + */ + +#include "libcamera/internal/converter/converter_dw100.h" + +#include + +#include + +#include "libcamera/internal/media_device.h" +#include "libcamera/internal/v4l2_videodevice.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(Converter) + +/** + * \class libcamera::ConverterDW100 + * \brief The i.MX8MP dewarp converter implements the converter interface based + * on V4L2 M2M device. +*/ + +/** + * \fn ConverterDW100::ConverterDW100 + * \brief Construct a ConverterDW100 instance + * \param[in] media The media device implementing the converter + */ +ConverterDW100::ConverterDW100(std::shared_ptr media) + : V4L2M2MConverter(media.get(), Feature::InputCrop) +{ +} + +} /* namespace libcamera */ diff --git a/src/libcamera/converter/meson.build b/src/libcamera/converter/meson.build index 2aa72fe4..175581b8 100644 --- a/src/libcamera/converter/meson.build +++ b/src/libcamera/converter/meson.build @@ -1,5 +1,6 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_sources += files([ + 'converter_dw100.cpp', 'converter_v4l2_m2m.cpp' ]) From patchwork Fri Jul 26 11:47:15 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 20714 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 4B70BBDC71 for ; Fri, 26 Jul 2024 11:47:43 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id AB8F86337E; Fri, 26 Jul 2024 13:47:42 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="t4eEXgtE"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id AE97D6336B for ; Fri, 26 Jul 2024 13:47:33 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B493983F; Fri, 26 Jul 2024 13:46:48 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1721994409; bh=rOebwUw8mptPzhY3dggJ8H7dvqP41iw+t1zbpPXXXh0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=t4eEXgtEfuz+cLs+56hwHYzOrdLYYDz0KGDRE8q8NVWZJmrLGZD3ua9bmggl/f6wJ xtpB7c4q6OrIsmr3JJ60Y1Kh2w351UE/BDBSX+q/Jsv+E47BSIX1Ruav69KXSHHIU+ coqnmzvYNFQrFMoZHKe7zZdAw1QIWBLdrzzEDSfc= From: Umang Jain To: libcamera-devel@lists.libcamera.org Cc: Umang Jain Subject: [PATCH v6 5/5] libcamera: rkisp1: Plumb the ConverterDW100 converter Date: Fri, 26 Jul 2024 17:17:15 +0530 Message-ID: <20240726114715.226468-6-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.45.0 In-Reply-To: <20240726114715.226468-1-umang.jain@ideasonboard.com> References: <20240726114715.226468-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Plumb the ConverterDW100 converter in the rkisp1 pipeline handler. If the dewarper is found, it is instantiated and buffers are exported from it, instead of RkISP1Path. Internal buffers are allocated for the RkISP1Path in case where dewarper is going to be used. The RKISP1 pipeline handler now supports scaler crop control through ConverterDW100. Register the ScalerCrop control for the cameras created in the RKISP1 pipeline handler. Signed-off-by: Umang Jain --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 154 ++++++++++++++++++++++- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index 25f2cc97..32ec0fdf 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -8,9 +8,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -33,6 +35,7 @@ #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/converter/converter_dw100.h" #include "libcamera/internal/delayed_controls.h" #include "libcamera/internal/device_enumerator.h" #include "libcamera/internal/framebuffer.h" @@ -62,6 +65,8 @@ struct RkISP1FrameInfo { bool paramDequeued; bool metadataProcessed; + + std::optional scalerCrop; }; class RkISP1Frames @@ -181,6 +186,7 @@ private: void bufferReady(FrameBuffer *buffer); void paramReady(FrameBuffer *buffer); void statReady(FrameBuffer *buffer); + void dewarpBufferReady(FrameBuffer *buffer); void frameStart(uint32_t sequence); int allocateBuffers(Camera *camera); @@ -200,6 +206,13 @@ private: RkISP1MainPath mainPath_; RkISP1SelfPath selfPath_; + std::unique_ptr dewarper_; + bool useDewarper_; + + /* Internal buffers used when dewarper is being used */ + std::vector> mainPathBuffers_; + std::queue availableMainPathBuffers_; + std::vector> paramBuffers_; std::vector> statBuffers_; std::queue availableParamBuffers_; @@ -222,6 +235,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req FrameBuffer *paramBuffer = nullptr; FrameBuffer *statBuffer = nullptr; + FrameBuffer *mainPathBuffer = nullptr; + FrameBuffer *selfPathBuffer = nullptr; if (!isRaw) { if (pipe_->availableParamBuffers_.empty()) { @@ -239,10 +254,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req statBuffer = pipe_->availableStatBuffers_.front(); pipe_->availableStatBuffers_.pop(); + + if (pipe_->useDewarper_) { + mainPathBuffer = pipe_->availableMainPathBuffers_.front(); + pipe_->availableMainPathBuffers_.pop(); + } } - FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_); - FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_); + if (!mainPathBuffer) + mainPathBuffer = request->findBuffer(&data->mainPathStream_); + selfPathBuffer = request->findBuffer(&data->selfPathStream_); RkISP1FrameInfo *info = new RkISP1FrameInfo; @@ -268,6 +289,7 @@ int RkISP1Frames::destroy(unsigned int frame) pipe_->availableParamBuffers_.push(info->paramBuffer); pipe_->availableStatBuffers_.push(info->statBuffer); + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); frameInfo_.erase(info->frame); @@ -283,6 +305,7 @@ void RkISP1Frames::clear() pipe_->availableParamBuffers_.push(info->paramBuffer); pipe_->availableStatBuffers_.push(info->statBuffer); + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); delete info; } @@ -607,7 +630,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() */ PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) - : PipelineHandler(manager), hasSelfPath_(true) + : PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false) { } @@ -785,12 +808,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) << " crop " << rect; std::map streamConfig; + std::vector> outputCfgs; for (const StreamConfiguration &cfg : *config) { if (cfg.stream() == &data->mainPathStream_) { ret = mainPath_.configure(cfg, format); streamConfig[0] = IPAStream(cfg.pixelFormat, cfg.size); + /* Configure dewarp */ + if (dewarper_ && !isRaw_) { + outputCfgs.push_back(const_cast(cfg)); + ret = dewarper_->configure(cfg, outputCfgs); + useDewarper_ = ret ? false : true; + } } else if (hasSelfPath_) { ret = selfPath_.configure(cfg, format); streamConfig[1] = IPAStream(cfg.pixelFormat, @@ -839,6 +869,9 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S RkISP1CameraData *data = cameraData(camera); unsigned int count = stream->configuration().bufferCount; + if (useDewarper_) + return dewarper_->exportBuffers(&data->mainPathStream_, count, buffers); + if (stream == &data->mainPathStream_) return mainPath_.exportBuffers(count, buffers); else if (hasSelfPath_ && stream == &data->selfPathStream_) @@ -866,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) ret = stat_->allocateBuffers(maxCount, &statBuffers_); if (ret < 0) goto error; + + /* If the dewarper is being used, allocate internal buffers for ISP */ + if (useDewarper_) { + ret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_); + if (ret < 0) + goto error; + + for (std::unique_ptr &buffer : mainPathBuffers_) + availableMainPathBuffers_.push(buffer.get()); + } } for (std::unique_ptr &buffer : paramBuffers_) { @@ -889,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) error: paramBuffers_.clear(); statBuffers_.clear(); + mainPathBuffers_.clear(); return ret; } @@ -903,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera) while (!availableParamBuffers_.empty()) availableParamBuffers_.pop(); + while (!availableMainPathBuffers_.empty()) + availableMainPathBuffers_.pop(); + paramBuffers_.clear(); statBuffers_.clear(); + mainPathBuffers_.clear(); std::vector ids; for (IPABuffer &ipabuf : data->ipaBuffers_) @@ -961,6 +1009,14 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL << "Failed to start statistics " << camera->id(); return ret; } + + if (useDewarper_) { + ret = dewarper_->start(); + if (ret) { + LOG(RkISP1, Error) << "Failed to start dewarper"; + return ret; + } + } } if (data->mainPath_->isEnabled()) { @@ -1015,6 +1071,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera) if (ret) LOG(RkISP1, Warning) << "Failed to stop parameters for " << camera->id(); + + if (useDewarper_) + dewarper_->stop(); } ASSERT(data->queuedRequests_.empty()); @@ -1045,6 +1104,25 @@ int PipelineHandlerRkISP1::queueRequestDevice(Camera *camera, Request *request) info->paramBuffer->cookie()); } + const auto &crop = request->controls().get(controls::ScalerCrop); + if (crop && useDewarper_) { + Rectangle appliedRect = crop.value(); + int ret = dewarper_->setInputCrop(&data->mainPathStream_, &appliedRect); + if (!ret) { + if (appliedRect != crop.value()) { + /* + * \todo How to handle these case? + * Do we aim for pixel perfect set rectangles? + */ + LOG(RkISP1, Warning) + << "Applied rectangle " << appliedRect.toString() + << " differs from requested " << crop.value().toString(); + } + + info->scalerCrop = appliedRect; + } + } + data->frame_++; return 0; @@ -1110,6 +1188,12 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data) { ControlInfoMap::Map rkisp1Controls; + if (dewarper_) { + auto [minCrop, maxCrop] = dewarper_->inputCropBounds(&data->mainPathStream_); + + rkisp1Controls[&controls::ScalerCrop] = ControlInfo(minCrop, maxCrop, maxCrop); + } + /* Add the IPA registered controls to list of camera controls. */ for (const auto &ipaControl : data->ipaControls_) rkisp1Controls[ipaControl.first] = ipaControl.second; @@ -1173,6 +1257,7 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) { + std::shared_ptr dwpMediaDevice; const MediaPad *pad; DeviceMatch dm("rkisp1"); @@ -1250,6 +1335,26 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); + /* If dewarper is present, create its instance. */ + DeviceMatch dwp("dw100"); + dwp.add("dw100-source"); + dwp.add("dw100-sink"); + + dwpMediaDevice = enumerator->search(dwp); + if (dwpMediaDevice) { + dewarper_ = std::make_unique(std::move(dwpMediaDevice)); + if (dewarper_->isValid()) { + dewarper_->outputBufferReady.connect( + this, &PipelineHandlerRkISP1::dewarpBufferReady); + + LOG(RkISP1, Info) << "Using DW100 dewarper " << dewarper_->deviceNode(); + } else { + LOG(RkISP1, Debug) << "Found DW100 dewarper " << dewarper_->deviceNode() + << " but invalid"; + dewarper_.reset(); + } + } + /* * Enumerate all sensors connected to the ISP and create one * camera instance for each of them. @@ -1296,7 +1401,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) return; const FrameMetadata &metadata = buffer->metadata(); - Request *request = buffer->request(); + Request *request = info->request; if (metadata.status != FrameMetadata::FrameCancelled) { /* @@ -1313,11 +1418,52 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) data->delayedCtrls_->get(metadata.sequence); data->ipa_->processStatsBuffer(info->frame, 0, ctrls); } + } else { if (isRaw_) info->metadataProcessed = true; } + if (useDewarper_) { + /* Do not queue cancelled frames to dewarper. */ + if (metadata.status == FrameMetadata::FrameCancelled) { + for (auto it : request->buffers()) + completeBuffer(request, it.second); + + tryCompleteRequest(info); + return; + } + + /* + * Queue input and output buffers to the dewarper. The output + * buffers for the dewarper are the buffers of the request, supplied + * by the application. + */ + int ret = dewarper_->queueBuffers(buffer, request->buffers()); + if (ret < 0) + LOG(RkISP1, Error) << "Cannot queue buffers to dewarper: " + << strerror(-ret); + + return; + } + + completeBuffer(request, buffer); + tryCompleteRequest(info); +} + +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + Request *request = buffer->request(); + + RkISP1FrameInfo *info = data->frameInfo_.find(buffer->request()); + if (!info) + return; + + if (info->scalerCrop) + request->metadata().set(controls::ScalerCrop, info->scalerCrop.value()); + completeBuffer(request, buffer); tryCompleteRequest(info); }