From patchwork Thu Nov 11 10:49:05 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Hanlin Chen X-Patchwork-Id: 14548 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 44E78BDB1C for ; Thu, 11 Nov 2021 10:49:37 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 0072760361; Thu, 11 Nov 2021 11:49:37 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="B3xDokDH"; dkim-atps=neutral Received: from mail-pl1-x636.google.com (mail-pl1-x636.google.com [IPv6:2607:f8b0:4864:20::636]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7FF1E6032C for ; Thu, 11 Nov 2021 11:49:35 +0100 (CET) Received: by mail-pl1-x636.google.com with SMTP id u11so5491594plf.3 for ; Thu, 11 Nov 2021 02:49:35 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=rdnEleC34M0zr+gS5rQKJvmxxwea1e6rZmRlToyr4Ik=; b=B3xDokDHVzh4frwnNBeZCl18NWgflfCFzA8jG0rlT0TI1kpcOwlkj+lJN/7qx/5X+/ KqfQGgg0mmglkjGqdFZcWLt3g4kuufpeffKt8SiR1xeUPiO0sU4tiDdnMFmPaec2FQi3 0ahIBJFK0VHFnUmmLMvlKEmhLaLYMo7UfZrxA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=rdnEleC34M0zr+gS5rQKJvmxxwea1e6rZmRlToyr4Ik=; b=qujM2e292yo/diOYHJ75HOp+UoODY5/h3lt8lhtnUWsNazpn1XW71R7/aDunpKtYWH p2UieOmDuCIGfUBkyZpLcNDstEQfS75L+qoiI5vU7oldlxHeKgpLNo8CIRWLak3QY7Ta 4H5SUKd7fqYkH8RM96SITgkpfCFiVOPf62dEPLdtdAljwDMrfbH80CapfP6Cl1/sFUGf v4H7M/mS+l4QnUuQkN1/oz1AgF2IPTG0FEQhmISyP6dj86AsEiZtMLK13T+9WqBgtlJc XffkfHmjGQEFlz7Ugcz8ioSZAhKfgCPjNlbjPAxOq6skCKGRFrhuFi2QvHa1zQkCYE0U Zyfg== X-Gm-Message-State: AOAM530TW++cuB1EDPho+/Um/7RcEOaMRliFXuuQaNH16ts4EGi1jcn7 Zk5yWJ1ewt8JcNEYId478ugvvzYMJrcbxg== X-Google-Smtp-Source: ABdhPJyabafKp9MqRFwvpPHkm5JnBuBxRKmJDY9V9jSI4HnWr2a2A7nRTTIdRpyD4JWgNOvU3cl8YA== X-Received: by 2002:a17:902:c7d5:b0:143:72b7:2ca5 with SMTP id r21-20020a170902c7d500b0014372b72ca5mr6602592pla.20.1636627773810; Thu, 11 Nov 2021 02:49:33 -0800 (PST) Received: from localhost ([2401:fa00:1:10:3925:da9c:3049:be7a]) by smtp.gmail.com with UTF8SMTPSA id fs21sm7444307pjb.1.2021.11.11.02.49.33 (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128); Thu, 11 Nov 2021 02:49:33 -0800 (PST) From: Han-Lin Chen To: libcamera-devel@lists.libcamera.org Date: Thu, 11 Nov 2021 18:49:05 +0800 Message-Id: <20211111104908.295992-4-hanlinchen@chromium.org> X-Mailer: git-send-email 2.34.0.rc1.387.gb447b232ab-goog In-Reply-To: <20211111104908.295992-1-hanlinchen@chromium.org> References: <20211111104908.295992-1-hanlinchen@chromium.org> MIME-Version: 1.0 Subject: [libcamera-devel] [IPU3-IPA PATCH v3 3/6] ipu3: Set statistics with the effective AE AiqResults X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Set the statistics with the latest AE AiqResults which has the same exposure time and analog gain. The patch reduces the AE hunting during the converging process. Signed-off-by: Han-Lin Chen Reviewed-by: Kieran Bingham --- aiq/aiq_input_parameters.cpp | 2 +- ipu3.cpp | 75 ++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/aiq/aiq_input_parameters.cpp b/aiq/aiq_input_parameters.cpp index bc87b31..46553a6 100644 --- a/aiq/aiq_input_parameters.cpp +++ b/aiq/aiq_input_parameters.cpp @@ -130,7 +130,7 @@ AiqInputParameters &AiqInputParameters::operator=(const AiqInputParameters &othe void AiqInputParameters::setAeAwbAfDefaults() { - /*Ae Params */ + /* Ae Params */ aeInputParams.num_exposures = NUM_EXPOSURES; aeInputParams.frame_use = ia_aiq_frame_use_preview; aeInputParams.flash_mode = ia_aiq_flash_mode_off; diff --git a/ipu3.cpp b/ipu3.cpp index 8126e9d..9d07268 100644 --- a/ipu3.cpp +++ b/ipu3.cpp @@ -59,7 +59,8 @@ private: void fillParams(unsigned int frame, ipu3_uapi_params *params); void parseStatistics(unsigned int frame, int64_t frameTimestamp, - const ipu3_uapi_stats_3a *stats); + const ipu3_uapi_stats_3a *stats, + const ControlList& sensorCtrls); void setControls(unsigned int frame); @@ -83,7 +84,7 @@ private: /* Temporary storage until we have a FrameContext object / struct */ aiq::AiqInputParameters aiqInputParams_; - aiq::AiqResults results_; + aiq::AiqResultsRingBuffer resultsHistory_; BinaryData aiqb_; BinaryData nvm_; @@ -282,6 +283,8 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo, /* Upate the camera controls using the new sensor settings. */ updateControls(sensorInfo_, ctrls_, ipaControls); + resultsHistory_.reset(); + return 0; } @@ -327,7 +330,10 @@ void IPAIPU3::processEvent(const IPU3Event &event) const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); - parseStatistics(event.frame, event.frameTimestamp, stats); + parseStatistics(event.frame, + event.frameTimestamp, + stats, + event.sensorControls); break; } case EventFillParams: { @@ -374,14 +380,16 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) */ /* Run algorithms into/using this context structure */ - if (frame % 10 == 0) - aiq_.run2a(frame, aiqInputParams_, results_); + resultsHistory_.extendOne(); + aiq::AiqResults& latestResults = resultsHistory_.latest(); - aic_.updateRuntimeParams(results_); + aiq_.run2a(frame, aiqInputParams_, latestResults); + aic_.updateRuntimeParams(latestResults); aic_.run(params); - exposure_ = results_.ae()->exposures[0].sensor_exposure->coarse_integration_time; - gain_ = results_.ae()->exposures[0].sensor_exposure->analog_gain_code_global; + exposure_ = latestResults.ae()->exposures[0].sensor_exposure->coarse_integration_time; + gain_ = latestResults.ae()->exposures[0].sensor_exposure->analog_gain_code_global; + setControls(frame); IPU3Action op; @@ -392,7 +400,8 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) void IPAIPU3::parseStatistics(unsigned int frame, int64_t frameTimestamp, - const ipu3_uapi_stats_3a *stats) + const ipu3_uapi_stats_3a *stats, + const ControlList& sensorCtrls) { ControlList ctrls(controls::controls); @@ -403,10 +412,45 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ ASSERT (frameTimestamp > 0); - aiq_.setStatistics(frame, frameTimestamp, results_, stats); + + /* Ae algorithm expects the statistics to be set with its corresponding + * Ae result, i.e., the Ae result should match the exposure time and + * analog gain with the the effective sensor controls of the statistics. + * Search the required Ae result in the result history and combine it + * with the latest result as the input to AIQ::setStatistics(). + */ + + int32_t effectiveExpo = 0; + int32_t effectiveGain = 0; + ControlValue ctrlValue; + + ctrlValue = sensorCtrls.get(V4L2_CID_EXPOSURE); + if (!ctrlValue.isNone()) + effectiveExpo = ctrlValue.get(); + + ctrlValue = sensorCtrls.get(V4L2_CID_ANALOGUE_GAIN); + if (!ctrlValue.isNone()) + effectiveGain = ctrlValue.get(); + + auto pred = [effectiveExpo, effectiveGain] (aiq::AiqResults& result) { + ia_aiq_exposure_sensor_parameters* sensorExposure = + result.ae()->exposures[0].sensor_exposure; + + return (effectiveExpo == sensorExposure->coarse_integration_time || + effectiveGain == sensorExposure->analog_gain_code_global); + }; + + aiq::AiqResults& latestResults = resultsHistory_.latest(); + aiq::AiqResults& aeMatchedResults = resultsHistory_.searchBackward(pred, latestResults); + + aiq::AiqResults combinedResults = latestResults; + combinedResults.setAe(aeMatchedResults.ae()); + + /* Aiq library expects timestamp in microseconds */ + aiq_.setStatistics(frame, (frameTimestamp / 1000), combinedResults, stats); /* Set frame durations from exposure results */ - ia_aiq_exposure_sensor_parameters *sensorExposure = results_.ae()->exposures->sensor_exposure; + ia_aiq_exposure_sensor_parameters *sensorExposure = combinedResults.ae()->exposures->sensor_exposure; int64_t frameDuration = (sensorExposure->line_length_pixels * sensorExposure->frame_length_lines) / (sensorInfo_.pixelRate / 1e6); ctrls.set(controls::FrameDuration, frameDuration); @@ -423,10 +467,11 @@ void IPAIPU3::setControls(unsigned int frame) IPU3Action op; op.op = ActionSetSensorControls; - ControlList ctrls(ctrls_); - ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); - ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_)); - op.controls = ctrls; + ControlList sensorCtrls(ctrls_); + sensorCtrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); + sensorCtrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_)); + + op.sensorControls = sensorCtrls; queueFrameAction.emit(frame, op); }