[{"id":20987,"web_url":"https://patchwork.libcamera.org/comment/20987/","msgid":"<163717148041.420308.8346709288256434086@Monstersaurus>","date":"2021-11-17T17:51:20","subject":"Re: [libcamera-devel] [IPU3-IPA PATCH v3 3/6] ipu3: Set statistics\n\twith the effective AE AiqResults","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Han-Lin Chen (2021-11-11 10:49:05)\n> Set the statistics with the latest AE AiqResults which has the same\n> exposure time and analog gain. The patch reduces the AE hunting during\n> the converging process.\n> \n> Signed-off-by: Han-Lin Chen \n> ---\n> aiq/aiq_input_parameters.cpp | 2 +-\n> ipu3.cpp | 75 ++++++++++++++++++++++++++++--------\n> 2 files changed, 61 insertions(+), 16 deletions(-)\n> \n> diff --git a/aiq/aiq_input_parameters.cpp b/aiq/aiq_input_parameters.cpp\n> index bc87b31..46553a6 100644\n> --- a/aiq/aiq_input_parameters.cpp\n> +++ b/aiq/aiq_input_parameters.cpp\n> @@ -130,7 +130,7 @@ AiqInputParameters &AiqInputParameters::operator=(const AiqInputParameters &othe\n> \n> void AiqInputParameters::setAeAwbAfDefaults()\n> {\n> - /*Ae Params */\n> + /* Ae Params */\n> aeInputParams.num_exposures = NUM_EXPOSURES;\n> aeInputParams.frame_use = ia_aiq_frame_use_preview;\n> aeInputParams.flash_mode = ia_aiq_flash_mode_off;\n> diff --git a/ipu3.cpp b/ipu3.cpp\n> index 8126e9d..9d07268 100644\n> --- a/ipu3.cpp\n> +++ b/ipu3.cpp\n> @@ -59,7 +59,8 @@ private:\n> void fillParams(unsigned int frame, ipu3_uapi_params *params);\n> void parseStatistics(unsigned int frame,\n> int64_t frameTimestamp,\n> - const ipu3_uapi_stats_3a *stats);\n> + const ipu3_uapi_stats_3a *stats,\n> + const ControlList& sensorCtrls);\n> \n> void setControls(unsigned int frame);\n> \n> @@ -83,7 +84,7 @@ private:\n> \n> /* Temporary storage until we have a FrameContext object / struct */\n> aiq::AiqInputParameters aiqInputParams_;\n> - aiq::AiqResults results_;\n> + aiq::AiqResultsRingBuffer resultsHistory_;\n> \n> BinaryData aiqb_;\n> BinaryData nvm_;\n> @@ -282,6 +283,8 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo,\n> /* Upate the camera controls using the new sensor settings. */\n> updateControls(sensorInfo_, ctrls_, ipaControls);\n> \n> + resultsHistory_.reset();\n> +\n> return 0;\n> }\n> \n> @@ -327,7 +330,10 @@ void IPAIPU3::processEvent(const IPU3Event &event)\n> const ipu3_uapi_stats_3a *stats =\n> reinterpret_cast(mem.data());\n> \n> - parseStatistics(event.frame, event.frameTimestamp, stats);\n> + parseStatistics(event.frame,\n> + event.frameTimestamp,\n> + stats,\n> + event.sensorControls);\n> break;\n> }\n> case EventFillParams: {\n> @@ -374,14 +380,16 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)\n> */\n> \n> /* Run algorithms into/using this context structure */\n> - if (frame % 10 == 0)\n> - aiq_.run2a(frame, aiqInputParams_, results_);\n> + resultsHistory_.extendOne();\n> + aiq::AiqResults& latestResults = resultsHistory_.latest();\n> \n> - aic_.updateRuntimeParams(results_);\n> + aiq_.run2a(frame, aiqInputParams_, latestResults);\n> + aic_.updateRuntimeParams(latestResults);\n> aic_.run(params);\n> \n> - exposure_ = results_.ae()->exposures[0].sensor_exposure->coarse_integration_time;\n> - gain_ = results_.ae()->exposures[0].sensor_exposure->analog_gain_code_global;\n> + exposure_ = latestResults.ae()->exposures[0].sensor_exposure->coarse_integration_time;\n> + gain_ = latestResults.ae()->exposures[0].sensor_exposure->analog_gain_code_global;\n> +\n> setControls(frame);\n> \n> IPU3Action op;\n> @@ -392,7 +400,8 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)\n> \n> void IPAIPU3::parseStatistics(unsigned int frame,\n> int64_t frameTimestamp,\n> - const ipu3_uapi_stats_3a *stats)\n> + const ipu3_uapi_stats_3a *stats,\n> + const ControlList& sensorCtrls)\n> {\n> ControlList ctrls(controls::controls);\n> \n> @@ -403,10 +412,45 @@ void IPAIPU3::parseStatistics(unsigned int frame,\n> */\n> \n> ASSERT (frameTimestamp > 0);\n> - aiq_.setStatistics(frame, frameTimestamp, results_, stats);\n> +\n> + /* Ae algorithm expects the statistics to be set with its corresponding\n> + * Ae result, i.e., the Ae result should match the exposure time and\n> + * analog gain with the the effective sensor controls of the statistics.\n> + * Search the required Ae result in the result history and combine it\n> + * with the latest result as the input to AIQ::setStatistics().\n> + */\n> +\n> + int32_t effectiveExpo = 0;\n> + int32_t effectiveGain = 0;\n> + ControlValue ctrlValue;\n> +\n> + ctrlValue = sensorCtrls.get(V4L2_CID_EXPOSURE);\n> + if (!ctrlValue.isNone())\n> + effectiveExpo = ctrlValue.get();\n> +\n> + ctrlValue = sensorCtrls.get(V4L2_CID_ANALOGUE_GAIN);\n> + if (!ctrlValue.isNone())\n> + effectiveGain = ctrlValue.get();\n> +\n> + auto pred = [effectiveExpo, effectiveGain] (aiq::AiqResults& result) {\n> + ia_aiq_exposure_sensor_parameters* sensorExposure =\n> + result.ae()->exposures[0].sensor_exposure;\n> +\n> + return (effectiveExpo == sensorExposure->coarse_integration_time ||\n> + effectiveGain == sensorExposure->analog_gain_code_global);\n> + };\n> +\n> + aiq::AiqResults& latestResults = resultsHistory_.latest();\n> + aiq::AiqResults& aeMatchedResults = resultsHistory_.searchBackward(pred, latestResults);\n> +\n> + aiq::AiqResults combinedResults = latestResults;\n> + combinedResults.setAe(aeMatchedResults.ae());\n> +\n> + /* Aiq library expects timestamp in microseconds */\n\nThat's a specific bug fix, that could/should have had it's own patch.\n\n> + aiq_.setStatistics(frame, (frameTimestamp / 1000), combinedResults, stats);\n\nbrackets aren't needed there either, but it does help improve\nreadability to me anyway.\n\n\n> /* Set frame durations from exposure results */\n> - ia_aiq_exposure_sensor_parameters *sensorExposure = results_.ae()->exposures->sensor_exposure;\n> + ia_aiq_exposure_sensor_parameters *sensorExposure = combinedResults.ae()->exposures->sensor_exposure;\n> int64_t frameDuration = (sensorExposure->line_length_pixels * sensorExposure->frame_length_lines) /\n> (sensorInfo_.pixelRate / 1e6);\n> ctrls.set(controls::FrameDuration, frameDuration);\n> @@ -423,10 +467,11 @@ void IPAIPU3::setControls(unsigned int frame)\n> IPU3Action op;\n> op.op = ActionSetSensorControls;\n> \n> - ControlList ctrls(ctrls_);\n> - ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_));\n> - ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_));\n> - op.controls = ctrls;\n> + ControlList sensorCtrls(ctrls_);\n> + sensorCtrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_));\n> + sensorCtrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_));\n> +\n> + op.sensorControls = sensorCtrls;\n\nHrm .. I think this is crucially needed too now that we've merged the \n\nf12efa673064 ipa: ipu3: Extend ipu3 ipa interface for sensor controls\n\nSo that should have been a separate patch too.\n\n\nAnyway, patch splitting aside.\n\n\nReviewed-by: Kieran Bingham \n\n> \n> queueFrameAction.emit(frame, op);\n> }\n> -- \n> 2.34.0.rc1.387.gb447b232ab-goog\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 48812BDB1C\n\tfor ;\n\tWed, 17 Nov 2021 17:51:25 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 99C9F60376;\n\tWed, 17 Nov 2021 18:51:24 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 636D060121\n\tfor ;\n\tWed, 17 Nov 2021 18:51:23 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 006332CF;\n\tWed, 17 Nov 2021 18:51:22 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"KL/hfJLN\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1637171483;\n\tbh=fKzZ9QksjCsB6GLj0sGmawTpI1W72+bdg5HrPEU5NDc=;\n\th=In-Reply-To:References:Subject:From:To:Date:From;\n\tb=KL/hfJLNI8ffxlYCK5wUvTAqzm4wRv/4byiUWxFseIAIza3ntZzoygx7r0Iqs8uN+\n\tX4iqgECu6CIbhEL7HxVUEfRQizsWTJkpVomFoU8MbhfF4EYkd2D+mhqsE97yGCUKFS\n\tPz5TgkCO99of1D2ACrd29v+QgOWTja9YSF44DsEI=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20211111104908.295992-4-hanlinchen@chromium.org>","References":"<20211111104908.295992-1-hanlinchen@chromium.org>\n\t<20211111104908.295992-4-hanlinchen@chromium.org>","From":"Kieran Bingham ","To":"Han-Lin Chen ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Wed, 17 Nov 2021 17:51:20 +0000","Message-ID":"<163717148041.420308.8346709288256434086@Monstersaurus>","User-Agent":"alot/0.10","Subject":"Re: [libcamera-devel] [IPU3-IPA PATCH v3 3/6] ipu3: Set statistics\n\twith the effective AE AiqResults","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]