Patch Detail
Show a patch.
GET /api/1.1/patches/12226/?format=api
{ "id": 12226, "url": "https://patchwork.libcamera.org/api/1.1/patches/12226/?format=api", "web_url": "https://patchwork.libcamera.org/patch/12226/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210507084042.31879-6-naush@raspberrypi.com>", "date": "2021-05-07T08:40:42", "name": "[libcamera-devel,v6,5/5] ipa: raspberrypi: Rate-limit the controller algorithms", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "dc5c84f6f30f25420e43321f11a11b7eb1fbf098", "submitter": { "id": 34, "url": "https://patchwork.libcamera.org/api/1.1/people/34/?format=api", "name": "Naushir Patuck", "email": "naush@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/12226/mbox/", "series": [ { "id": 2010, "url": "https://patchwork.libcamera.org/api/1.1/series/2010/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2010", "date": "2021-05-07T08:40:37", "name": "ipa: raspberrypi: Rate-limit the controller algorithms", "version": 6, "mbox": "https://patchwork.libcamera.org/series/2010/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/12226/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/12226/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 0631ABF835\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 7 May 2021 08:40:56 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id BD98B6890C;\n\tFri, 7 May 2021 10:40:55 +0200 (CEST)", "from mail-wm1-x333.google.com (mail-wm1-x333.google.com\n\t[IPv6:2a00:1450:4864:20::333])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id C245668921\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 7 May 2021 10:40:52 +0200 (CEST)", "by mail-wm1-x333.google.com with SMTP id\n\tl18-20020a1ced120000b029014c1adff1edso6668371wmh.4\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 07 May 2021 01:40:52 -0700 (PDT)", "from naush-laptop.pitowers.org\n\t([2a00:1098:3142:14:81ab:95ee:9715:e9fb])\n\tby smtp.gmail.com with ESMTPSA id\n\tl12sm7604098wrm.76.2021.05.07.01.40.51\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tFri, 07 May 2021 01:40:51 -0700 (PDT)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"TfVgXges\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=from:to:cc:subject:date:message-id:in-reply-to:references\n\t:mime-version:content-transfer-encoding;\n\tbh=4S717ZsySRiN4h+aZW4nhYMwXVC+cgf8UZnjbQ6phO8=;\n\tb=TfVgXgesW084VhTrb1+XqCWAT9vl1OaSWlVcNYTp6Z2VfmLtUQGrHL5uEW+mPpm5lV\n\txEz3yQObidj4IFM8C/PmZF4q0WZB6LO8qUukgXqyTrVD8bpk5UgsSu9w0GuDliuZF8uT\n\tbAHQS2HWagUJSUFogXo/UYAzw0kmRKdIxCP66Dwf2Qz9DOX2Go6tyqmapw15kJFfu5Ws\n\tARHCR63JlsH8pIUaRF0Ies/QhCcTk2i5tznIYfsTJ3KFYb9jhx8GGBRXDk9WtywZohIm\n\tFB4J5ViPR6r/TxTzoyvDPtTYabC+/iZSpOMOtHwkOXDExHgMAQUGReh+XcmFcyRKF8OC\n\t0Lig==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:mime-version:content-transfer-encoding;\n\tbh=4S717ZsySRiN4h+aZW4nhYMwXVC+cgf8UZnjbQ6phO8=;\n\tb=mTll8zYt3/Gp3dozyZVMQ6XhXhpkbxUcT4eQiX5TQb9y8ZEfkPyu5D/Ue9RTIMtnl0\n\tzv0ZIcexhJwGayX/dIA9Z808v3a4Aj+lisUI4R/mWuZzFkUTkg3bqrd0QyOQK2wDppF5\n\tcbOvt5/1mcznYafAx4tMWNMgyng5UtMKEmV2sWx6+9ToFRu9/HBFZV4iMWKxnS2v88cT\n\t3qq42vHfyFjM2zPrCDQ+ElNcpSac0QMpOB54AUnuAFiocwWLtyP/TrnONOMWTX+RxsIP\n\tAzj63VDTEQHz2CvkzMWDhH6GkusuDy5B3Yll8YvcNKnrc3Gy0QFddoRlhRkj112UlScV\n\t81Ow==", "X-Gm-Message-State": "AOAM531I2klwNxW11A83i5QoZlZ4ppo910l6rDN6ShCXidBxKmV1Mfe4\n\thu7FZJ3NOUWfnbOjVl51v/MreAqXz4nH9Q==", "X-Google-Smtp-Source": "ABdhPJxeL3uesUMTy12RipniR9k9xi0PJmpYBu6vvZI2Xuzn1c8ZBeJEHz9TRSKB4gKmvDA22r5vnA==", "X-Received": "by 2002:a7b:c74d:: with SMTP id w13mr8455645wmk.25.1620376852053;\n\tFri, 07 May 2021 01:40:52 -0700 (PDT)", "From": "Naushir Patuck <naush@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 7 May 2021 09:40:42 +0100", "Message-Id": "<20210507084042.31879-6-naush@raspberrypi.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20210507084042.31879-1-naush@raspberrypi.com>", "References": "<20210507084042.31879-1-naush@raspberrypi.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v6 5/5] ipa: raspberrypi: Rate-limit the\n\tcontroller algorithms", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "The controller algorithms currently run on every frame provided to the\nIPA by the pipeline handler. This may be undesirable for very fast fps\noperating modes where it could significantly increase the computation\ncycles (per unit time) without providing any significant changes to the\nIQ parameters. The added latencies could also cause dropped frames.\n\nPass the FrameBuffer timestamp to the IPA through the controls. This\ntimestamp will be used to rate-limit the controller algorithms to run\nwith a minimum inter-frame time given by a compile time constant,\ncurrently set to 16.66ms. On startup, we don't rate-limit the algorithms\nuntil after the number of frames required for convergence.\n\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\n---\n src/ipa/raspberrypi/raspberrypi.cpp | 48 +++++++++++++++++--\n .../pipeline/raspberrypi/raspberrypi.cpp | 5 ++\n 2 files changed, 49 insertions(+), 4 deletions(-)", "diff": "diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp\nindex f6d1ab16a290..57f5497a6e6b 100644\n--- a/src/ipa/raspberrypi/raspberrypi.cpp\n+++ b/src/ipa/raspberrypi/raspberrypi.cpp\n@@ -61,6 +61,14 @@ constexpr unsigned int DefaultExposureTime = 20000;\n constexpr double defaultMinFrameDuration = 1e6 / 30.0;\n constexpr double defaultMaxFrameDuration = 1e6 / 0.01;\n \n+/*\n+ * Determine the minimum allowable inter-frame duration (in us) to run the\n+ * controller algorithms. If the pipeline handler provider frames at a rate\n+ * higher than this, we rate-limit the controller Prepare() and Process() calls\n+ * to lower than or equal to this rate.\n+ */\n+constexpr double controllerMinFrameDuration = 1e6 / 60.0;\n+\n LOG_DEFINE_CATEGORY(IPARPI)\n \n class IPARPi : public ipa::RPi::IPARPiInterface\n@@ -68,7 +76,7 @@ class IPARPi : public ipa::RPi::IPARPiInterface\n public:\n \tIPARPi()\n \t\t: controller_(), frameCount_(0), checkCount_(0), mistrustCount_(0),\n-\t\t lsTable_(nullptr), firstStart_(true)\n+\t\t lastRunTimestamp_(0), lsTable_(nullptr), firstStart_(true)\n \t{\n \t}\n \n@@ -146,6 +154,12 @@ private:\n \t/* Number of frames that need to be dropped on startup. */\n \tunsigned int dropFrameCount_;\n \n+\t/* Frame timestamp for the last run of the controller. */\n+\tuint64_t lastRunTimestamp_;\n+\n+\t/* Do we run a Controller::process() for this frame? */\n+\tbool processPending_;\n+\n \t/* LS table allocation passed in from the pipeline handler. */\n \tFileDescriptor lsTableHandle_;\n \tvoid *lsTable_;\n@@ -262,6 +276,7 @@ void IPARPi::start(const ControlList &controls, ipa::RPi::StartConfig *startConf\n \tstartConfig->dropFrameCount = dropFrameCount_;\n \n \tfirstStart_ = false;\n+\tlastRunTimestamp_ = 0;\n }\n \n void IPARPi::setMode(const CameraSensorInfo &sensorInfo)\n@@ -406,7 +421,7 @@ void IPARPi::signalStatReady(uint32_t bufferId)\n {\n \tif (++checkCount_ != frameCount_) /* assert here? */\n \t\tLOG(IPARPI, Error) << \"WARNING: Prepare/Process mismatch!!!\";\n-\tif (frameCount_ > mistrustCount_)\n+\tif (processPending_ && frameCount_ > mistrustCount_)\n \t\tprocessStats(bufferId);\n \n \treportMetadata();\n@@ -894,10 +909,11 @@ void IPARPi::returnEmbeddedBuffer(unsigned int bufferId)\n \n void IPARPi::prepareISP(const ipa::RPi::ISPConfig &data)\n {\n+\tint64_t frameTimestamp = data.controls.get(controls::draft::SensorTimestamp);\n+\tRPiController::Metadata lastMetadata;\n \tSpan<uint8_t> embeddedBuffer;\n \n-\trpiMetadata_.Clear();\n-\n+\tlastMetadata = std::move(rpiMetadata_);\n \tfillDeviceStatus(data.controls);\n \n \tif (data.embeddedBufferPresent) {\n@@ -920,6 +936,30 @@ void IPARPi::prepareISP(const ipa::RPi::ISPConfig &data)\n \tif (data.embeddedBufferPresent)\n \t\treturnEmbeddedBuffer(data.embeddedBufferId);\n \n+\t/* Allow a 10% margin on the comparison below. */\n+\tconstexpr double eps = controllerMinFrameDuration * 1e3 * 0.1;\n+\tif (lastRunTimestamp_ && frameCount_ > dropFrameCount_ &&\n+\t frameTimestamp - lastRunTimestamp_ + eps < controllerMinFrameDuration * 1e3) {\n+\t\t/*\n+\t\t * Ensure we update the controller metadata with the new frame's\n+\t\t * exposure/gain values so that the correct values are returned\n+\t\t * out in libcamera metadata later on. Any other bits of metadata\n+\t\t * that were added in helper_->Prepare() will also be moved across.\n+\t\t * All other metadata values must remain the same as the last frame.\n+\t\t *\n+\t\t * We do this by overwriting the current frame's metadata into the\n+\t\t * previous frame's metadata object, and then swapping the latter\n+\t\t * with the former.\n+\t\t */\n+\t\tlastMetadata.Overwrite(rpiMetadata_);\n+\t\tstd::swap(rpiMetadata_, lastMetadata);\n+\t\tprocessPending_ = false;\n+\t\treturn;\n+\t}\n+\n+\tlastRunTimestamp_ = frameTimestamp;\n+\tprocessPending_ = true;\n+\n \tControlList ctrls(ispCtrls_);\n \n \tcontroller_.Prepare(&rpiMetadata_);\ndiff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\nindex 2a917455500f..9cf9c8c6cebd 100644\n--- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n+++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n@@ -1414,6 +1414,11 @@ void RPiCameraData::unicamBufferDequeue(FrameBuffer *buffer)\n \t\t * DelayedControl and queue them along with the frame buffer.\n \t\t */\n \t\tControlList ctrl = delayedCtrls_->get(buffer->metadata().sequence);\n+\t\t/*\n+\t\t * Add the frame timestamp to the ControlList for the IPA to use\n+\t\t * as it does not receive the FrameBuffer object.\n+\t\t */\n+\t\tctrl.set(controls::draft::SensorTimestamp, buffer->metadata().timestamp);\n \t\tbayerQueue_.push({ buffer, std::move(ctrl) });\n \t} else {\n \t\tembeddedQueue_.push(buffer);\n", "prefixes": [ "libcamera-devel", "v6", "5/5" ] }