[{"id":31421,"web_url":"https://patchwork.libcamera.org/comment/31421/","msgid":"","date":"2024-09-26T19:25:34","subject":"Re: [PATCH v7 16/18] libcamera: software_isp: Use DelayedControls","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi Milan,\n\nOn 21/09/24 12:01 am, Milan Zamazal wrote:\n> Use the standard libcamera mechanism to report the \"current\" controls\n> rather than delaying updates by counting from the last update.\n>\n> A problem is that with software ISP we cannot be sure about the sensor\n> delay. The original implementation simply skips exposure updates for 2\n> frames, which should be enough in most cases. After this change, we\n> assume the delay being exactly 2 frames, which may or may not be correct\n> and may work with outdated values if the delay is shorter.\n>\n> According to Kieran, the wrong parts are also wrong on the\n> IPU3/RKISP1/Mali pipelines and only RPi have this correct. We need to\n> fix this, by correctly specifying the gains in the libipa camera sensor\n> helpers. The sooner the better because this change could introduce a\n> risk of increasing oscillations.\n>\n> This patch also prepares moving exposure+gain to an algorithm module\n> where the original delay mechanism would be a (possibly unnecessary)\n> complication.\n>\n> Resolves software ISP TODO #11 + #12.\n>\n> Signed-off-by: Milan Zamazal \n> Reviewed-by: Kieran Bingham \n\nReviewed-by: Umang Jain \n> ---\n> src/ipa/simple/soft_simple.cpp | 16 +------------\n> src/libcamera/pipeline/simple/simple.cpp | 18 ++++++++++++---\n> src/libcamera/software_isp/TODO | 29 ------------------------\n> 3 files changed, 16 insertions(+), 47 deletions(-)\n>\n> diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp\n> index f8d923c5..60310a8e 100644\n> --- a/src/ipa/simple/soft_simple.cpp\n> +++ b/src/ipa/simple/soft_simple.cpp\n> @@ -59,8 +59,7 @@ class IPASoftSimple : public ipa::soft::IPASoftInterface, public Module\n> public:\n> \tIPASoftSimple()\n> \t\t: params_(nullptr), stats_(nullptr),\n> -\t\t context_({ {}, {}, { kMaxFrameContexts } }),\n> -\t\t ignoreUpdates_(0)\n> +\t\t context_({ {}, {}, { kMaxFrameContexts } })\n> \t{\n> \t}\n> \n> @@ -98,7 +97,6 @@ private:\n> \tint32_t exposure_;\n> \tdouble againMin_, againMax_, againMinStep_;\n> \tdouble again_;\n> -\tunsigned int ignoreUpdates_;\n> };\n> \n> IPASoftSimple::~IPASoftSimple()\n> @@ -298,16 +296,6 @@ void IPASoftSimple::processStats(const uint32_t frame,\n> \n> \t/* \\todo Switch to the libipa/algorithm.h API someday. */\n> \n> -\t/*\n> -\t * AE / AGC, use 2 frames delay to make sure that the exposure and\n> -\t * the gain set have applied to the camera sensor.\n> -\t * \\todo This could be handled better with DelayedControls.\n> -\t */\n> -\tif (ignoreUpdates_ > 0) {\n> -\t\t--ignoreUpdates_;\n> -\t\treturn;\n> -\t}\n> -\n> \t/*\n> \t * Calculate Mean Sample Value (MSV) according to formula from:\n> \t * https://www.araa.asn.au/acra/acra2007/papers/paper84final.pdf\n> @@ -356,8 +344,6 @@ void IPASoftSimple::processStats(const uint32_t frame,\n> \tctrls.set(V4L2_CID_ANALOGUE_GAIN,\n> \t\t static_cast(camHelper_ ? camHelper_->gainCode(again_) : again_));\n> \n> -\tignoreUpdates_ = 2;\n> -\n> \tsetSensorControls.emit(ctrls);\n> \n> \tLOG(IPASoft, Debug) << \"exposureMSV \" << exposureMSV\n> diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp\n> index 61157fe6..1e13bd74 100644\n> --- a/src/libcamera/pipeline/simple/simple.cpp\n> +++ b/src/libcamera/pipeline/simple/simple.cpp\n> @@ -32,6 +32,7 @@\n> #include \"libcamera/internal/camera.h\"\n> #include \"libcamera/internal/camera_sensor.h\"\n> #include \"libcamera/internal/converter.h\"\n> +#include \"libcamera/internal/delayed_controls.h\"\n> #include \"libcamera/internal/device_enumerator.h\"\n> #include \"libcamera/internal/media_device.h\"\n> #include \"libcamera/internal/pipeline_handler.h\"\n> @@ -278,6 +279,8 @@ public:\n> \tstd::vector configs_;\n> \tstd::map> formats_;\n> \n> +\tstd::unique_ptr delayedCtrls_;\n> +\n> \tstd::vector> conversionBuffers_;\n> \tstd::queue> conversionQueue_;\n> \tbool useConversion_;\n> @@ -896,14 +899,13 @@ void SimpleCameraData::conversionOutputDone(FrameBuffer *buffer)\n> \n> void SimpleCameraData::ispStatsReady(uint32_t frame, uint32_t bufferId)\n> {\n> -\t/* \\todo Use the DelayedControls class */\n> \tswIsp_->processStats(frame, bufferId,\n> -\t\t\t sensor_->getControls({ V4L2_CID_ANALOGUE_GAIN,\n> -\t\t\t\t\t\t V4L2_CID_EXPOSURE }));\n> +\t\t\t delayedCtrls_->get(frame));\n> }\n> \n> void SimpleCameraData::setSensorControls(const ControlList &sensorControls)\n> {\n> +\tdelayedCtrls_->push(sensorControls);\n> \tControlList ctrls(sensorControls);\n> \tsensor_->setControls(&ctrls);\n> }\n> @@ -1284,6 +1286,16 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c)\n> \tif (outputCfgs.empty())\n> \t\treturn 0;\n> \n> +\tstd::unordered_map params = {\n> +\t\t{ V4L2_CID_ANALOGUE_GAIN, { 2, false } },\n> +\t\t{ V4L2_CID_EXPOSURE, { 2, false } },\n> +\t};\n> +\tdata->delayedCtrls_ =\n> +\t\tstd::make_unique(data->sensor_->device(),\n> +\t\t\t\t\t\t params);\n> +\tdata->video_->frameStart.connect(data->delayedCtrls_.get(),\n> +\t\t\t\t\t &DelayedControls::applyControls);\n> +\n> \tStreamConfiguration inputCfg;\n> \tinputCfg.pixelFormat = pipeConfig->captureFormat;\n> \tinputCfg.size = pipeConfig->captureSize;\n> diff --git a/src/libcamera/software_isp/TODO b/src/libcamera/software_isp/TODO\n> index 9978afc0..8eeede46 100644\n> --- a/src/libcamera/software_isp/TODO\n> +++ b/src/libcamera/software_isp/TODO\n> @@ -209,35 +209,6 @@ At some point, yes.\n> \n> ---\n> \n> -11. Improve handling the sensor controls which take effect with a delay\n> -\n> -> void IPASoftSimple::processStats(const ControlList &sensorControls)\n> -> {\n> -> ...\n> ->\t/*\n> ->\t * AE / AGC, use 2 frames delay to make sure that the exposure and\n> ->\t * the gain set have applied to the camera sensor.\n> ->\t */\n> ->\tif (ignore_updates_ > 0) {\n> ->\t\t--ignore_updates_;\n> ->\t\treturn;\n> ->\t}\n> -\n> -This could be handled better with DelayedControls.\n> -\n> ----\n> -\n> -12. Use DelayedControls class in ispStatsReady()\n> -\n> -> void SimpleCameraData::ispStatsReady()\n> -> {\n> -> \tswIsp_->processStats(sensor_->getControls({ V4L2_CID_ANALOGUE_GAIN,\n> -> \t\t\t\t\t\t V4L2_CID_EXPOSURE }));\n> -\n> -You should use the DelayedControls class.\n> -\n> ----\n> -\n> 13. Improve black level and colour gains application\n> \n> I think the black level should eventually be moved before debayering, and","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 976CEC0F1B\n\tfor ;\n\tThu, 26 Sep 2024 19:25:41 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3D6616350F;\n\tThu, 26 Sep 2024 21:25:41 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 1C042618DA\n\tfor ;\n\tThu, 26 Sep 2024 21:25:39 +0200 (CEST)","from [IPV6:2405:201:2015:f873:c173:4b:4a04:3a21] (unknown\n\t[IPv6:2405:201:2015:f873:c173:4b:4a04:3a21])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 0E9EE8D4;\n\tThu, 26 Sep 2024 21:24:09 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"pEmPMo2G\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1727378650;\n\tbh=0eI/39QRbmhb3CUZUhFHYMmz6wRM+IWNQ4GjstiWjjc=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=pEmPMo2GRvjctlxpvvo4jL4mX9iP5CwtgsOURzm1VjR75PmF6sA6Jdx8jxshbfmbR\n\tgH1Z+yGBGht6hb29MCCSVGhVcUCDu3mXYfZIgAizXc+vnpuFWymSdDqtuN8+1XF3y1\n\t9olxvnQeOxm9fx04qlfml65ZlLWUCWnQnqD2IYA4=","Message-ID":"","Date":"Fri, 27 Sep 2024 00:55:34 +0530","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [PATCH v7 16/18] libcamera: software_isp: Use DelayedControls","To":"Milan Zamazal , libcamera-devel@lists.libcamera.org","Cc":"Laurent Pinchart ,\n\tDaniel Scally ,\n\tKieran Bingham ","References":"<20240920183143.1674117-1-mzamazal@redhat.com>\n\t<20240920183143.1674117-17-mzamazal@redhat.com>","Content-Language":"en-US","From":"Umang Jain ","In-Reply-To":"<20240920183143.1674117-17-mzamazal@redhat.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]