Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 21924,
    "url": "https://patchwork.libcamera.org/api/patches/21924/?format=api",
    "web_url": "https://patchwork.libcamera.org/patch/21924/",
    "project": {
        "id": 1,
        "url": "https://patchwork.libcamera.org/api/projects/1/?format=api",
        "name": "libcamera",
        "link_name": "libcamera",
        "list_id": "libcamera_core",
        "list_email": "libcamera-devel@lists.libcamera.org",
        "web_url": "",
        "scm_url": "",
        "webscm_url": ""
    },
    "msgid": "<20241115122540.478103-7-dan.scally@ideasonboard.com>",
    "date": "2024-11-15T12:25:35",
    "name": "[v4,06/11] mali-c55: Plumb the IPA module in",
    "commit_ref": null,
    "pull_url": null,
    "state": "accepted",
    "archived": false,
    "hash": "a783099eb927ce01c375b23930a27a36ab0ae7ae",
    "submitter": {
        "id": 156,
        "url": "https://patchwork.libcamera.org/api/people/156/?format=api",
        "name": "Dan Scally",
        "email": "dan.scally@ideasonboard.com"
    },
    "delegate": null,
    "mbox": "https://patchwork.libcamera.org/patch/21924/mbox/",
    "series": [
        {
            "id": 4794,
            "url": "https://patchwork.libcamera.org/api/series/4794/?format=api",
            "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4794",
            "date": "2024-11-15T12:25:29",
            "name": "Add Mali-C55 IPA Module and Algorithms",
            "version": 4,
            "mbox": "https://patchwork.libcamera.org/series/4794/mbox/"
        }
    ],
    "comments": "https://patchwork.libcamera.org/api/patches/21924/comments/",
    "check": "pending",
    "checks": "https://patchwork.libcamera.org/api/patches/21924/checks/",
    "tags": {},
    "headers": {
        "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>",
        "X-Original-To": "parsemail@patchwork.libcamera.org",
        "Delivered-To": "parsemail@patchwork.libcamera.org",
        "Received": [
            "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4DC89C0F1B\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 15 Nov 2024 12:26:49 +0000 (UTC)",
            "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 234BE65892;\n\tFri, 15 Nov 2024 13:26:45 +0100 (CET)",
            "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9F1F265889\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 15 Nov 2024 13:26:35 +0100 (CET)",
            "from mail.ideasonboard.com\n\t(cpc141996-chfd3-2-0-cust928.12-3.cable.virginm.net [86.13.91.161])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id ECB0D1114;\n\tFri, 15 Nov 2024 13:26:20 +0100 (CET)"
        ],
        "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"SyoMTd3o\"; dkim-atps=neutral",
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1731673581;\n\tbh=fH5WDY30SnHAqO8biibBJeMSAoSXJg4MUPuLTZYZfe8=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=SyoMTd3or1mghJ7kzV8WiLrZk28MpJ36j+UPJP4hFM5rGQrXWlKjjgZBvgS5vUFsA\n\tnb6uL42N+qlpsCzocbzinbCP6lq1G9QiE2bOIouWH3CqWke2UcMskh689WSg4OSfl+\n\tem7BBQCNqyEt7mF51mF3ZaNfx1QIdb0R5s3aQCws=",
        "From": "Daniel Scally <dan.scally@ideasonboard.com>",
        "To": "libcamera-devel@lists.libcamera.org",
        "Cc": "Anthony.McGivern@arm.com, Jacopo Mondi <jacopo.mondi@ideasonboard.com>, \n\tNayden Kanchev <nayden.kanchev@arm.com>,\n\tDaniel Scally <dan.scally@ideasonboard.com>",
        "Subject": "[PATCH v4 06/11] mali-c55: Plumb the IPA module in",
        "Date": "Fri, 15 Nov 2024 12:25:35 +0000",
        "Message-Id": "<20241115122540.478103-7-dan.scally@ideasonboard.com>",
        "X-Mailer": "git-send-email 2.34.1",
        "In-Reply-To": "<20241115122540.478103-1-dan.scally@ideasonboard.com>",
        "References": "<20241115122540.478103-1-dan.scally@ideasonboard.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "X-BeenThere": "libcamera-devel@lists.libcamera.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "<libcamera-devel.lists.libcamera.org>",
        "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>",
        "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>",
        "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>",
        "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>",
        "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>",
        "Errors-To": "libcamera-devel-bounces@lists.libcamera.org",
        "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"
    },
    "content": "From: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\n\nPlumb the Pipeline-IPA loop in.\n\nLoad the IPA module at camera creation time and create the loop between\nthe pipeline and the IPA.\n\nWhen a new Request is queued the IPA is asked to prepare the parameters\nbuffer, once ready it notifies the pipeline which queues the parameters\nto the ISP along with a buffer for statistics and frames,\n\nOnce statistics are ready they get passed to the IPA which upates its\nsettings for the next frame.\n\nDriveby fix an error message in the Pipeline Handler's ::freeBuffers()\nfunction which reported a problem with the wrong video device in an\nerror path.\n\nAcked-by: Nayden Kanchev  <nayden.kanchev@arm.com>\nSigned-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\nSigned-off-by: Daniel Scally <dan.scally@ideasonboard.com>\n---\nChanges in v4:\n\n\t- None\n\nChanges in v3:\n\n\t- Corrected the call to ipa_->configurationFile() to include a fallback\n\t  file name.\n\t- Used the new CameraSensor::getSensorDelays() to fetch sensor delay\n\t  values from CameraSensorProperties.\nChanges in v2:\n\n\t- None\n\n src/libcamera/pipeline/mali-c55/mali-c55.cpp | 421 +++++++++++++++++--\n 1 file changed, 378 insertions(+), 43 deletions(-)",
    "diff": "diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\nindex e451204b..43ef0572 100644\n--- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n+++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n@@ -21,17 +21,23 @@\n #include <libcamera/camera.h>\n #include <libcamera/formats.h>\n #include <libcamera/geometry.h>\n+#include <libcamera/property_ids.h>\n #include <libcamera/stream.h>\n \n #include <libcamera/ipa/core_ipa_interface.h>\n+#include <libcamera/ipa/mali-c55_ipa_interface.h>\n+#include <libcamera/ipa/mali-c55_ipa_proxy.h>\n \n #include \"libcamera/internal/bayer_format.h\"\n #include \"libcamera/internal/camera.h\"\n #include \"libcamera/internal/camera_sensor.h\"\n+#include \"libcamera/internal/delayed_controls.h\"\n #include \"libcamera/internal/device_enumerator.h\"\n #include \"libcamera/internal/framebuffer.h\"\n+#include \"libcamera/internal/ipa_manager.h\"\n #include \"libcamera/internal/media_device.h\"\n #include \"libcamera/internal/pipeline_handler.h\"\n+#include \"libcamera/internal/request.h\"\n #include \"libcamera/internal/v4l2_subdevice.h\"\n #include \"libcamera/internal/v4l2_videodevice.h\"\n \n@@ -72,6 +78,16 @@ constexpr Size kMaliC55MinSize = { 128, 128 };\n constexpr Size kMaliC55MaxSize = { 8192, 8192 };\n constexpr unsigned int kMaliC55ISPInternalFormat = MEDIA_BUS_FMT_RGB121212_1X36;\n \n+struct MaliC55FrameInfo {\n+\tRequest *request;\n+\n+\tFrameBuffer *paramBuffer;\n+\tFrameBuffer *statBuffer;\n+\n+\tbool paramsDone;\n+\tbool statsDone;\n+};\n+\n class MaliC55CameraData : public Camera::Private\n {\n public:\n@@ -81,6 +97,7 @@ public:\n \t}\n \n \tint init();\n+\tint loadIPA();\n \n \t/* Deflect these functionalities to either TPG or CameraSensor. */\n \tconst std::vector<Size> sizes(unsigned int mbusCode) const;\n@@ -89,7 +106,7 @@ public:\n \tint pixfmtToMbusCode(const PixelFormat &pixFmt) const;\n \tconst PixelFormat &bestRawFormat() const;\n \n-\tvoid updateControls();\n+\tvoid updateControls(const ControlInfoMap &ipaControls);\n \n \tPixelFormat adjustRawFormat(const PixelFormat &pixFmt) const;\n \tSize adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const;\n@@ -102,8 +119,15 @@ public:\n \tStream frStream_;\n \tStream dsStream_;\n \n+\tstd::unique_ptr<ipa::mali_c55::IPAProxyMaliC55> ipa_;\n+\tstd::vector<IPABuffer> ipaStatBuffers_;\n+\tstd::vector<IPABuffer> ipaParamBuffers_;\n+\n+\tstd::unique_ptr<DelayedControls> delayedCtrls_;\n+\n private:\n \tvoid initTPGData();\n+\tvoid setSensorControls(const ControlList &sensorControls);\n \n \tstd::string id_;\n \tstd::vector<unsigned int> tpgCodes_;\n@@ -167,6 +191,11 @@ void MaliC55CameraData::initTPGData()\n \ttpgResolution_ = tpgSizes_.back();\n }\n \n+void MaliC55CameraData::setSensorControls(const ControlList &sensorControls)\n+{\n+\tdelayedCtrls_->push(sensorControls);\n+}\n+\n const std::vector<Size> MaliC55CameraData::sizes(unsigned int mbusCode) const\n {\n \tif (sensor_)\n@@ -272,7 +301,7 @@ const PixelFormat &MaliC55CameraData::bestRawFormat() const\n \treturn invalidPixFmt;\n }\n \n-void MaliC55CameraData::updateControls()\n+void MaliC55CameraData::updateControls(const ControlInfoMap &ipaControls)\n {\n \tif (!sensor_)\n \t\treturn;\n@@ -290,6 +319,9 @@ void MaliC55CameraData::updateControls()\n \t\tControlInfo(ispMinCrop, sensorInfo.analogCrop,\n \t\t\t    sensorInfo.analogCrop);\n \n+\tfor (auto const &c : ipaControls)\n+\t\tcontrols.emplace(c.first, c.second);\n+\n \tcontrolInfo_ = ControlInfoMap(std::move(controls), controls::controls);\n }\n \n@@ -343,6 +375,45 @@ Size MaliC55CameraData::adjustRawSizes(const PixelFormat &rawFmt, const Size &si\n \treturn bestSize;\n }\n \n+int MaliC55CameraData::loadIPA()\n+{\n+\tint ret;\n+\n+\t/* Do not initialize IPA for TPG. */\n+\tif (!sensor_)\n+\t\treturn 0;\n+\n+\tipa_ = IPAManager::createIPA<ipa::mali_c55::IPAProxyMaliC55>(pipe(), 1, 1);\n+\tif (!ipa_)\n+\t\treturn -ENOENT;\n+\n+\tipa_->setSensorControls.connect(this, &MaliC55CameraData::setSensorControls);\n+\n+\tstd::string ipaTuningFile = ipa_->configurationFile(sensor_->model() + \".yaml\",\n+\t\t\t\t\t\t\t    \"uncalibrated.yaml\");\n+\n+\t/* We need to inform the IPA of the sensor configuration */\n+\tipa::mali_c55::IPAConfigInfo ipaConfig{};\n+\n+\tret = sensor_->sensorInfo(&ipaConfig.sensorInfo);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tipaConfig.sensorControls = sensor_->controls();\n+\n+\tControlInfoMap ipaControls;\n+\tret = ipa_->init({ ipaTuningFile, sensor_->model() }, ipaConfig,\n+\t\t\t &ipaControls);\n+\tif (ret) {\n+\t\tLOG(MaliC55, Error) << \"Failed to initialise the Mali-C55 IPA\";\n+\t\treturn ret;\n+\t}\n+\n+\tupdateControls(ipaControls);\n+\n+\treturn 0;\n+}\n+\n class MaliC55CameraConfiguration : public CameraConfiguration\n {\n public:\n@@ -352,6 +423,7 @@ public:\n \t}\n \n \tStatus validate() override;\n+\tconst Transform &combinedTransform() { return combinedTransform_; }\n \n \tV4L2SubdeviceFormat sensorFormat_;\n \n@@ -359,6 +431,7 @@ private:\n \tstatic constexpr unsigned int kMaxStreams = 2;\n \n \tconst MaliC55CameraData *data_;\n+\tTransform combinedTransform_;\n };\n \n CameraConfiguration::Status MaliC55CameraConfiguration::validate()\n@@ -368,6 +441,19 @@ CameraConfiguration::Status MaliC55CameraConfiguration::validate()\n \tif (config_.empty())\n \t\treturn Invalid;\n \n+\t/*\n+\t * The TPG doesn't support flips, so we only need to calculate a\n+\t * transform if we have a sensor.\n+\t */\n+\tif (data_->sensor_) {\n+\t\tOrientation requestedOrientation = orientation;\n+\t\tcombinedTransform_ = data_->sensor_->computeTransform(&orientation);\n+\t\tif (orientation != requestedOrientation)\n+\t\t\tstatus = Adjusted;\n+\t} else {\n+\t\tcombinedTransform_ = Transform::Rot0;\n+\t}\n+\n \t/* Only 2 streams available. */\n \tif (config_.size() > kMaxStreams) {\n \t\tconfig_.resize(kMaxStreams);\n@@ -531,7 +617,10 @@ public:\n \tint queueRequestDevice(Camera *camera, Request *request) override;\n \n \tvoid imageBufferReady(FrameBuffer *buffer);\n+\tvoid paramsBufferReady(FrameBuffer *buffer);\n \tvoid statsBufferReady(FrameBuffer *buffer);\n+\tvoid paramsComputed(unsigned int requestId);\n+\tvoid statsProcessed(unsigned int requestId, const ControlList &metadata);\n \n \tbool match(DeviceEnumerator *enumerator) override;\n \n@@ -576,6 +665,10 @@ private:\n \t\t\tpipe.stream = nullptr;\n \t}\n \n+\tMaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);\n+\tMaliC55FrameInfo *findFrameInfo(Request *request);\n+\tvoid tryComplete(MaliC55FrameInfo *info);\n+\n \tint configureRawStream(MaliC55CameraData *data,\n \t\t\t       const StreamConfiguration &config,\n \t\t\t       V4L2SubdeviceFormat &subdevFormat);\n@@ -585,7 +678,7 @@ private:\n \n \tvoid applyScalerCrop(Camera *camera, const ControlList &controls);\n \n-\tvoid registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n+\tbool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n \t\t\t\tconst std::string &name);\n \tbool registerTPGCamera(MediaLink *link);\n \tbool registerSensorCamera(MediaLink *link);\n@@ -601,6 +694,8 @@ private:\n \tstd::vector<std::unique_ptr<FrameBuffer>> paramsBuffers_;\n \tstd::queue<FrameBuffer *> availableParamsBuffers_;\n \n+\tstd::map<unsigned int, MaliC55FrameInfo> frameInfoMap_;\n+\n \tstd::array<MaliC55Pipe, MaliC55NumPipes> pipes_;\n \n \tbool dsFitted_;\n@@ -849,6 +944,13 @@ int PipelineHandlerMaliC55::configure(Camera *camera,\n \tif (ret)\n \t\treturn ret;\n \n+\tif (data->sensor_) {\n+\t\tret = data->sensor_->setFormat(&subdevFormat,\n+\t\t\t\t\t       maliConfig->combinedTransform());\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n \tif (data->csi_) {\n \t\tret = data->csi_->setFormat(0, &subdevFormat);\n \t\tif (ret)\n@@ -930,7 +1032,55 @@ int PipelineHandlerMaliC55::configure(Camera *camera,\n \t\tpipe->stream = stream;\n \t}\n \n-\tdata->updateControls();\n+\tif (!data->ipa_)\n+\t\treturn 0;\n+\n+\t/*\n+\t * Enable the media link between the ISP subdevice and the statistics\n+\t * video device.\n+\t */\n+\tconst MediaEntity *ispEntity = isp_->entity();\n+\tret = ispEntity->getPadByIndex(3)->links()[0]->setEnabled(true);\n+\tif (ret) {\n+\t\tLOG(MaliC55, Error) << \"Couldn't enable statistics link\";\n+\t\treturn ret;\n+\t}\n+\n+\t/*\n+\t * Enable the media link between the ISP subdevice and the parameters\n+\t * video device.\n+\t */\n+\tret = ispEntity->getPadByIndex(4)->links()[0]->setEnabled(true);\n+\tif (ret) {\n+\t\tLOG(MaliC55, Error) << \"Couldn't enable parameters link\";\n+\t\treturn ret;\n+\t}\n+\n+\t/* We need to inform the IPA of the sensor configuration */\n+\tipa::mali_c55::IPAConfigInfo ipaConfig{};\n+\n+\tret = data->sensor_->sensorInfo(&ipaConfig.sensorInfo);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tipaConfig.sensorControls = data->sensor_->controls();\n+\n+\t/*\n+\t * And we also need to tell the IPA the bayerOrder of the data (as\n+\t * affected by any flips that we've configured)\n+\t */\n+\tconst Transform &combinedTransform = maliConfig->combinedTransform();\n+\tBayerFormat::Order bayerOrder = data->sensor_->bayerOrder(combinedTransform);\n+\n+\tControlInfoMap ipaControls;\n+\tret = data->ipa_->configure(ipaConfig, utils::to_underlying(bayerOrder),\n+\t\t\t\t    &ipaControls);\n+\tif (ret) {\n+\t\tLOG(MaliC55, Error) << \"Failed to configure IPA\";\n+\t\treturn ret;\n+\t}\n+\n+\tdata->updateControls(ipaControls);\n \n \treturn 0;\n }\n@@ -944,8 +1094,10 @@ int PipelineHandlerMaliC55::exportFrameBuffers(Camera *camera, Stream *stream,\n \treturn pipe->cap->exportBuffers(count, buffers);\n }\n \n-void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)\n+void PipelineHandlerMaliC55::freeBuffers(Camera *camera)\n {\n+\tMaliC55CameraData *data = cameraData(camera);\n+\n \twhile (!availableStatsBuffers_.empty())\n \t\tavailableStatsBuffers_.pop();\n \twhile (!availableParamsBuffers_.empty())\n@@ -954,11 +1106,18 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)\n \tstatsBuffers_.clear();\n \tparamsBuffers_.clear();\n \n+\tif (data->ipa_) {\n+\t\tdata->ipa_->unmapBuffers(data->ipaStatBuffers_);\n+\t\tdata->ipa_->unmapBuffers(data->ipaParamBuffers_);\n+\t}\n+\tdata->ipaStatBuffers_.clear();\n+\tdata->ipaParamBuffers_.clear();\n+\n \tif (stats_->releaseBuffers())\n \t\tLOG(MaliC55, Error) << \"Failed to release stats buffers\";\n \n \tif (params_->releaseBuffers())\n-\t\tLOG(MaliC55, Error) << \"Failed to release stats buffers\";\n+\t\tLOG(MaliC55, Error) << \"Failed to release params buffers\";\n \n \treturn;\n }\n@@ -966,6 +1125,7 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)\n int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)\n {\n \tMaliC55CameraData *data = cameraData(camera);\n+\tunsigned int ipaBufferId = 1;\n \tunsigned int bufferCount;\n \tint ret;\n \n@@ -978,27 +1138,51 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)\n \tif (ret < 0)\n \t\treturn ret;\n \n-\tfor (std::unique_ptr<FrameBuffer> &buffer : statsBuffers_)\n+\tfor (std::unique_ptr<FrameBuffer> &buffer : statsBuffers_) {\n+\t\tbuffer->setCookie(ipaBufferId++);\n+\t\tdata->ipaStatBuffers_.emplace_back(buffer->cookie(),\n+\t\t\t\t\t\t   buffer->planes());\n \t\tavailableStatsBuffers_.push(buffer.get());\n+\t}\n \n \tret = params_->allocateBuffers(bufferCount, &paramsBuffers_);\n \tif (ret < 0)\n \t\treturn ret;\n \n-\tfor (std::unique_ptr<FrameBuffer> &buffer : paramsBuffers_)\n+\tfor (std::unique_ptr<FrameBuffer> &buffer : paramsBuffers_) {\n+\t\tbuffer->setCookie(ipaBufferId++);\n+\t\tdata->ipaParamBuffers_.emplace_back(buffer->cookie(),\n+\t\t\t\t\t\t    buffer->planes());\n \t\tavailableParamsBuffers_.push(buffer.get());\n+\t}\n+\n+\tif (data->ipa_) {\n+\t\tdata->ipa_->mapBuffers(data->ipaStatBuffers_, true);\n+\t\tdata->ipa_->mapBuffers(data->ipaParamBuffers_, false);\n+\t}\n \n \treturn 0;\n }\n \n int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const ControlList *controls)\n {\n+\tMaliC55CameraData *data = cameraData(camera);\n \tint ret;\n \n \tret = allocateBuffers(camera);\n \tif (ret)\n \t\treturn ret;\n \n+\tif (data->ipa_) {\n+\t\tret = data->ipa_->start();\n+\t\tif (ret) {\n+\t\t\tLOG(MaliC55, Error)\n+\t\t\t\t<< \"Failed to start IPA\" << camera->id();\n+\t\t\tfreeBuffers(camera);\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n \tfor (MaliC55Pipe &pipe : pipes_) {\n \t\tif (!pipe.stream)\n \t\t\tcontinue;\n@@ -1008,6 +1192,8 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n \t\tret = pipe.cap->importBuffers(stream->configuration().bufferCount);\n \t\tif (ret) {\n \t\t\tLOG(MaliC55, Error) << \"Failed to import buffers\";\n+\t\t\tif (data->ipa_)\n+\t\t\t\tdata->ipa_->stop();\n \t\t\tfreeBuffers(camera);\n \t\t\treturn ret;\n \t\t}\n@@ -1015,6 +1201,8 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n \t\tret = pipe.cap->streamOn();\n \t\tif (ret) {\n \t\t\tLOG(MaliC55, Error) << \"Failed to start stream\";\n+\t\t\tif (data->ipa_)\n+\t\t\t\tdata->ipa_->stop();\n \t\t\tfreeBuffers(camera);\n \t\t\treturn ret;\n \t\t}\n@@ -1024,6 +1212,9 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n \tif (ret) {\n \t\tLOG(MaliC55, Error) << \"Failed to start stats stream\";\n \n+\t\tif (data->ipa_)\n+\t\t\tdata->ipa_->stop();\n+\n \t\tfor (MaliC55Pipe &pipe : pipes_) {\n \t\t\tif (pipe.stream)\n \t\t\t\tpipe.cap->streamOff();\n@@ -1038,6 +1229,8 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n \t\tLOG(MaliC55, Error) << \"Failed to start params stream\";\n \n \t\tstats_->streamOff();\n+\t\tif (data->ipa_)\n+\t\t\tdata->ipa_->stop();\n \n \t\tfor (MaliC55Pipe &pipe : pipes_) {\n \t\t\tif (pipe.stream)\n@@ -1048,11 +1241,19 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n \t\treturn ret;\n \t}\n \n+\tret = isp_->setFrameStartEnabled(true);\n+\tif (ret)\n+\t\tLOG(MaliC55, Error) << \"Failed to enable frame start events\";\n+\n \treturn 0;\n }\n \n-void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera)\n+void PipelineHandlerMaliC55::stopDevice(Camera *camera)\n {\n+\tMaliC55CameraData *data = cameraData(camera);\n+\n+\tisp_->setFrameStartEnabled(false);\n+\n \tfor (MaliC55Pipe &pipe : pipes_) {\n \t\tif (!pipe.stream)\n \t\t\tcontinue;\n@@ -1063,6 +1264,8 @@ void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera)\n \n \tstats_->streamOff();\n \tparams_->streamOff();\n+\tif (data->ipa_)\n+\t\tdata->ipa_->stop();\n \tfreeBuffers(camera);\n }\n \n@@ -1164,64 +1367,179 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,\n \n int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)\n {\n-\tFrameBuffer *statsBuffer;\n-\tint ret;\n+\tMaliC55CameraData *data = cameraData(camera);\n+\n+\t/* Do not run the IPA if the TPG is in use. */\n+\tif (!data->ipa_) {\n+\t\tMaliC55FrameInfo frameInfo;\n+\t\tframeInfo.request = request;\n+\t\tframeInfo.statBuffer = nullptr;\n+\t\tframeInfo.paramBuffer = nullptr;\n+\t\tframeInfo.paramsDone = true;\n+\t\tframeInfo.statsDone = true;\n+\n+\t\tframeInfoMap_[request->sequence()] = frameInfo;\n+\n+\t\tfor (auto &[stream, buffer] : request->buffers()) {\n+\t\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n+\n+\t\t\tpipe->cap->queueBuffer(buffer);\n+\t\t}\n+\n+\t\treturn 0;\n+\t}\n \n \tif (availableStatsBuffers_.empty()) {\n \t\tLOG(MaliC55, Error) << \"Stats buffer underrun\";\n \t\treturn -ENOENT;\n \t}\n \n-\tstatsBuffer = availableStatsBuffers_.front();\n+\tif (availableParamsBuffers_.empty()) {\n+\t\tLOG(MaliC55, Error) << \"Params buffer underrun\";\n+\t\treturn -ENOENT;\n+\t}\n+\n+\tMaliC55FrameInfo frameInfo;\n+\tframeInfo.request = request;\n+\n+\tframeInfo.statBuffer = availableStatsBuffers_.front();\n \tavailableStatsBuffers_.pop();\n+\tframeInfo.paramBuffer = availableParamsBuffers_.front();\n+\tavailableParamsBuffers_.pop();\n \n-\t/*\n-\t * We need to associate the Request to this buffer even though it's a\n-\t * purely internal one because we will need to use request->sequence()\n-\t * later.\n-\t */\n-\tstatsBuffer->_d()->setRequest(request);\n+\tframeInfo.paramsDone = false;\n+\tframeInfo.statsDone = false;\n \n-\tfor (auto &[stream, buffer] : request->buffers()) {\n-\t\tMaliC55Pipe *pipe = pipeFromStream(cameraData(camera), stream);\n+\tframeInfoMap_[request->sequence()] = frameInfo;\n \n-\t\tret = pipe->cap->queueBuffer(buffer);\n-\t\tif (ret)\n-\t\t\treturn ret;\n+\tdata->ipa_->queueRequest(request->sequence(), request->controls());\n+\tdata->ipa_->fillParams(request->sequence(),\n+\t\t\t       frameInfo.paramBuffer->cookie());\n+\n+\treturn 0;\n+}\n+\n+MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(Request *request)\n+{\n+\tfor (auto &[sequence, info] : frameInfoMap_) {\n+\t\tif (info.request == request)\n+\t\t\treturn &info;\n \t}\n \n-\t/*\n-\t * Some controls need to be applied immediately, as in example,\n-\t * the ScalerCrop one.\n-\t *\n-\t * \\todo Move it buffer queue time (likely after the IPA has filled in\n-\t * the parameters buffer) once we have plumbed the IPA loop in.\n-\t */\n-\tapplyScalerCrop(camera, request->controls());\n+\treturn nullptr;\n+}\n \n-\tret = stats_->queueBuffer(statsBuffer);\n-\tif (ret)\n-\t\treturn ret;\n+MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)\n+{\n+\tfor (auto &[sequence, info] : frameInfoMap_) {\n+\t\tif (info.paramBuffer == buffer ||\n+\t\t    info.statBuffer == buffer)\n+\t\t\treturn &info;\n+\t}\n \n-\treturn 0;\n+\treturn nullptr;\n+}\n+\n+void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)\n+{\n+\tif (!info->paramsDone)\n+\t\treturn;\n+\tif (!info->statsDone)\n+\t\treturn;\n+\n+\tRequest *request = info->request;\n+\tif (request->hasPendingBuffers())\n+\t\treturn;\n+\n+\tif (info->statBuffer)\n+\t\tavailableStatsBuffers_.push(info->statBuffer);\n+\tif (info->paramBuffer)\n+\t\tavailableParamsBuffers_.push(info->paramBuffer);\n+\n+\tframeInfoMap_.erase(request->sequence());\n+\n+\tcompleteRequest(request);\n }\n \n void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer)\n {\n \tRequest *request = buffer->request();\n+\tMaliC55FrameInfo *info = findFrameInfo(request);\n+\tASSERT(info);\n \n \tif (completeBuffer(request, buffer))\n-\t\tcompleteRequest(request);\n+\t\ttryComplete(info);\n+}\n+\n+void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)\n+{\n+\tMaliC55FrameInfo *info = findFrameInfo(buffer);\n+\tASSERT(info);\n+\n+\tinfo->paramsDone = true;\n+\n+\ttryComplete(info);\n }\n \n void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)\n {\n-\tavailableStatsBuffers_.push(buffer);\n+\tMaliC55FrameInfo *info = findFrameInfo(buffer);\n+\tASSERT(info);\n+\n+\tRequest *request = info->request;\n+\tMaliC55CameraData *data = cameraData(request->_d()->camera());\n+\n+\tControlList sensorControls = data->delayedCtrls_->get(buffer->metadata().sequence);\n+\n+\tdata->ipa_->processStats(request->sequence(), buffer->cookie(),\n+\t\t\t\t sensorControls);\n }\n \n-void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n+void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId)\n+{\n+\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n+\tRequest *request = frameInfo.request;\n+\tMaliC55CameraData *data = cameraData(request->_d()->camera());\n+\n+\t/*\n+\t * Queue buffers for stats and params, then queue buffers to the capture\n+\t * video devices.\n+\t */\n+\n+\tframeInfo.paramBuffer->_d()->metadata().planes()[0].bytesused =\n+\t\tsizeof(struct mali_c55_params_buffer);\n+\tparams_->queueBuffer(frameInfo.paramBuffer);\n+\tstats_->queueBuffer(frameInfo.statBuffer);\n+\n+\tfor (auto &[stream, buffer] : request->buffers()) {\n+\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n+\n+\t\tpipe->cap->queueBuffer(buffer);\n+\t}\n+}\n+\n+void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,\n+\t\t\t\t\t    const ControlList &metadata)\n+{\n+\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n+\n+\tframeInfo.statsDone = true;\n+\tframeInfo.request->metadata().merge(metadata);\n+\n+\ttryComplete(&frameInfo);\n+}\n+\n+bool PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n \t\t\t\t\t\tconst std::string &name)\n {\n+\tif (data->loadIPA())\n+\t\treturn false;\n+\n+\tif (data->ipa_) {\n+\t\tdata->ipa_->statsProcessed.connect(this, &PipelineHandlerMaliC55::statsProcessed);\n+\t\tdata->ipa_->paramsComputed.connect(this, &PipelineHandlerMaliC55::paramsComputed);\n+\t}\n+\n \tstd::set<Stream *> streams{ &data->frStream_ };\n \tif (dsFitted_)\n \t\tstreams.insert(&data->dsStream_);\n@@ -1229,6 +1547,8 @@ void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraDat\n \tstd::shared_ptr<Camera> camera = Camera::create(std::move(data),\n \t\t\t\t\t\t\tname, streams);\n \tregisterCamera(std::move(camera));\n+\n+\treturn true;\n }\n \n /*\n@@ -1254,9 +1574,7 @@ bool PipelineHandlerMaliC55::registerTPGCamera(MediaLink *link)\n \tif (data->init())\n \t\treturn false;\n \n-\tregisterMaliCamera(std::move(data), name);\n-\n-\treturn true;\n+\treturn registerMaliCamera(std::move(data), name);\n }\n \n /*\n@@ -1283,9 +1601,25 @@ bool PipelineHandlerMaliC55::registerSensorCamera(MediaLink *ispLink)\n \t\t\treturn false;\n \n \t\tdata->properties_ = data->sensor_->properties();\n-\t\tdata->updateControls();\n \n-\t\tregisterMaliCamera(std::move(data), sensor->name());\n+\t\tuint8_t exposureDelay, gainDelay, vblankDelay, hblankDelay;\n+\t\tdata->sensor_->getSensorDelays(exposureDelay, gainDelay,\n+\t\t\t\t\t       vblankDelay, hblankDelay);\n+\t\tstd::unordered_map<uint32_t, DelayedControls::ControlParams> params = {\n+\t\t\t{ V4L2_CID_ANALOGUE_GAIN, { gainDelay, false } },\n+\t\t\t{ V4L2_CID_EXPOSURE, { exposureDelay, false } },\n+\t\t};\n+\n+\t\tdata->delayedCtrls_ =\n+\t\t\tstd::make_unique<DelayedControls>(data->sensor_->device(),\n+\t\t\t\t\t\t\t  params);\n+\t\tisp_->frameStart.connect(data->delayedCtrls_.get(),\n+\t\t\t\t\t &DelayedControls::applyControls);\n+\n+\t\t/* \\todo: Init properties. */\n+\n+\t\tif (!registerMaliCamera(std::move(data), sensor->name()))\n+\t\t\treturn false;\n \t}\n \n \treturn true;\n@@ -1365,6 +1699,7 @@ bool PipelineHandlerMaliC55::match(DeviceEnumerator *enumerator)\n \t}\n \n \tstats_->bufferReady.connect(this, &PipelineHandlerMaliC55::statsBufferReady);\n+\tparams_->bufferReady.connect(this, &PipelineHandlerMaliC55::paramsBufferReady);\n \n \tispSink = isp_->entity()->getPadByIndex(0);\n \tif (!ispSink || ispSink->links().empty()) {\n",
    "prefixes": [
        "v4",
        "06/11"
    ]
}