From patchwork Thu Apr 15 10:18:43 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Naushir Patuck X-Patchwork-Id: 11948 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id A25D5BD224 for ; Thu, 15 Apr 2021 10:18:54 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 5ACC46880C; Thu, 15 Apr 2021 12:18:54 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=raspberrypi.com header.i=@raspberrypi.com header.b="sSGeYXsc"; dkim-atps=neutral Received: from mail-ej1-x62b.google.com (mail-ej1-x62b.google.com [IPv6:2a00:1450:4864:20::62b]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 51AD9605AE for ; Thu, 15 Apr 2021 12:18:53 +0200 (CEST) Received: by mail-ej1-x62b.google.com with SMTP id r9so36035399ejj.3 for ; Thu, 15 Apr 2021 03:18:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=raspberrypi.com; s=google; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=t7SfidY2iUUDfC9huQ5uIW1a/cX7q47udLj0SB/qTBk=; b=sSGeYXscd/P8oQvwMdJfoy2eS+DYCjAazhk1DFoOtETDQCX1T7JyFAWd4tJkdxgnC2 soOO7eTpA6E6mx1KnLvvxNM93nlDfn96XKkiRNnjpSwH0+GT6xqXgE6GQXfRkBNz2GGr Lio3IN7b9Y9999JmijWZ/UyyBpU/RYwIQ765+fm6f04kntX6HOMrj9Kdi6pTlUUTPK6w 1CyaiUTxZL0u9L+jLLUyI8nWoqWzTao7uvaXzcA8c0vGOuj8i72BYvzTgAZES5DfncD1 RdUbpfXmL3QRu06d3D+PJp5Aokq8JVMwzajPys+V4vDUSpLIvpDDDHqM9IOLYtbhwdDq ukJA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=t7SfidY2iUUDfC9huQ5uIW1a/cX7q47udLj0SB/qTBk=; b=Ftfj2FNJOk4gX+7zpedTudXhMVj1cZl0orgxMaOupKxc3MK15Q2Da3hElG74i1++5+ 4Ghx8tKHsW3guNyE+1kUBzox2D5l5bHpZRciXtHZAYDjwC7n5jCNRphDnfBVB/C0zhXh yBkkr8wq8Gf177jW2VNqtAMlq98DsDB1L4hPNHwmfrmRiMolcdjneZDjVLk5U+fDHWz1 BKYrtEGZmvbycMAWdnKzyj1HTszntE6W6wsN70i6QKKm/arzgNPEmsCF1+vWa/SOsKv1 ICfiLB79bttSahDABSuCUc11A0M9CVO18cHCRAvtLLtvc2n6zQYC6IiWtnA2Yhg+Iu/f i92Q== X-Gm-Message-State: AOAM533YfOn8MOXHI/sZp/VP3qVs7/oIV6nllJVCVgS5k7an0mqk47xt vQu5XUki7T8UHKyZR0UtP0e1GNP16r3JKQ== X-Google-Smtp-Source: ABdhPJxmughqgHfGk9V+GQszt9Lq9Lmj11FnCytZcbtVCVZtg5JtPEW+cbd+Q5t4KHtdnISabxBJBw== X-Received: by 2002:a17:906:4c91:: with SMTP id q17mr2707378eju.0.1618481932672; Thu, 15 Apr 2021 03:18:52 -0700 (PDT) Received: from naush-laptop.patuck.local ([88.97.76.4]) by smtp.gmail.com with ESMTPSA id c12sm2190144edx.54.2021.04.15.03.18.52 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 15 Apr 2021 03:18:52 -0700 (PDT) From: Naushir Patuck To: libcamera-devel@lists.libcamera.org Date: Thu, 15 Apr 2021 11:18:43 +0100 Message-Id: <20210415101843.1215926-1-naush@raspberrypi.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH] ipa: raspberrypi: Rate-limit the controller algorithms X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The controller algorithms currently run on every frame provided to the IPA by the pipeline handler. This may be undesirable for very fast fps operating modes where it could significantly increase the computation cycles (per unit time) without providing any significant changes to the IQ parameters. The added latencies could also cause dropped frames. Pass the FrameBuffer timestamp to the IPA through the controls. This timestamp will be used to rate-limit the controller algorithms to run with a minimum inter-frame time given by a compile time constant, currently set to 16.66ms. Signed-off-by: Naushir Patuck --- src/ipa/raspberrypi/raspberrypi.cpp | 39 ++++++++++++++++++- .../pipeline/raspberrypi/raspberrypi.cpp | 5 +++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp index dad6395f0823..aa071473d6e7 100644 --- a/src/ipa/raspberrypi/raspberrypi.cpp +++ b/src/ipa/raspberrypi/raspberrypi.cpp @@ -61,6 +61,14 @@ constexpr unsigned int DefaultExposureTime = 20000; constexpr double defaultMinFrameDuration = 1e6 / 30.0; constexpr double defaultMaxFrameDuration = 1e6 / 0.01; +/* + * Determine the minimum allowable inter-frame duration (in us) to run the + * controller algorithms. If the pipeline handler provider frames at a rate + * higher than this, we rate-limit the controller prepare() and process() calls + * to lower than or equal to this rate. + */ +constexpr double controllerMinFrameDuration = 1e6 / 60.0; + LOG_DEFINE_CATEGORY(IPARPI) class IPARPi : public ipa::RPi::IPARPiInterface @@ -68,7 +76,7 @@ class IPARPi : public ipa::RPi::IPARPiInterface public: IPARPi() : controller_(), frameCount_(0), checkCount_(0), mistrustCount_(0), - lsTable_(nullptr), firstStart_(true) + lastRunTimestamp_(0), lsTable_(nullptr), firstStart_(true) { } @@ -145,6 +153,12 @@ private: /* How many frames we should avoid running control algos on. */ unsigned int mistrustCount_; + /* Frame timestamp for the last run of the controller. */ + uint64_t lastRunTimestamp_; + + /* Do we run a Controller::process() for this frame? */ + bool processPending_; + /* LS table allocation passed in from the pipeline handler. */ FileDescriptor lsTableHandle_; void *lsTable_; @@ -406,7 +420,7 @@ void IPARPi::signalStatReady(uint32_t bufferId) { if (++checkCount_ != frameCount_) /* assert here? */ LOG(IPARPI, Error) << "WARNING: Prepare/Process mismatch!!!"; - if (frameCount_ > mistrustCount_) + if (processPending_ && frameCount_ > mistrustCount_) processStats(bufferId); reportMetadata(); @@ -894,6 +908,7 @@ void IPARPi::returnEmbeddedBuffer(unsigned int bufferId) void IPARPi::prepareISP(const ipa::RPi::ISPConfig &data) { + int64_t frameTimestamp = data.controls.get(controls::draft::SensorTimestamp); struct DeviceStatus deviceStatus = {}; bool success = false; @@ -919,6 +934,26 @@ void IPARPi::prepareISP(const ipa::RPi::ISPConfig &data) fillDeviceStatus(exposureLines, gainCode, deviceStatus); } + if (lastRunTimestamp_ && + frameTimestamp - lastRunTimestamp_ < controllerMinFrameDuration * 1e3) { + /* + * Ensure we update the controller metadata with the new frame's + * exposure/gain values so that the correct values are returned + * out in libcamera metadata later on. All other metadata values + * must remain the same as the last frame. + */ + rpiMetadata_.Set("device.status", deviceStatus); + processPending_ = false; + LOG(IPARPI, Debug) << "Rate-limiting the controller! inter-frame duration: " + << frameTimestamp - lastRunTimestamp_ + << ", min duration " + << controllerMinFrameDuration * 1e3; + return; + } + + lastRunTimestamp_ = frameTimestamp; + processPending_ = true; + ControlList ctrls(ispCtrls_); rpiMetadata_.Clear(); diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp index 2a917455500f..9cf9c8c6cebd 100644 --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp @@ -1414,6 +1414,11 @@ void RPiCameraData::unicamBufferDequeue(FrameBuffer *buffer) * DelayedControl and queue them along with the frame buffer. */ ControlList ctrl = delayedCtrls_->get(buffer->metadata().sequence); + /* + * Add the frame timestamp to the ControlList for the IPA to use + * as it does not receive the FrameBuffer object. + */ + ctrl.set(controls::draft::SensorTimestamp, buffer->metadata().timestamp); bayerQueue_.push({ buffer, std::move(ctrl) }); } else { embeddedQueue_.push(buffer);