Patch Detail
Show a patch.
GET /api/1.1/patches/10265/?format=api
{ "id": 10265, "url": "https://patchwork.libcamera.org/api/1.1/patches/10265/?format=api", "web_url": "https://patchwork.libcamera.org/patch/10265/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20201027110738.15751-1-david.plowman@raspberrypi.com>", "date": "2020-10-27T11:07:38", "name": "[libcamera-devel,v7,5/6] libcamera: pipeline: raspberrypi: Implementation of digital zoom", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "982e637c65ed0812e12ff1cfc240a286e100516a", "submitter": { "id": 42, "url": "https://patchwork.libcamera.org/api/1.1/people/42/?format=api", "name": "David Plowman", "email": "david.plowman@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/10265/mbox/", "series": [ { "id": 1419, "url": "https://patchwork.libcamera.org/api/1.1/series/1419/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1419", "date": "2020-10-27T11:07:38", "name": null, "version": 7, "mbox": "https://patchwork.libcamera.org/series/1419/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/10265/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/10265/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E0E68C3B5C\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 27 Oct 2020 11:07:48 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 72877620AE;\n\tTue, 27 Oct 2020 12:07:48 +0100 (CET)", "from mail-wr1-x436.google.com (mail-wr1-x436.google.com\n\t[IPv6:2a00:1450:4864:20::436])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 55CFF61059\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 27 Oct 2020 12:07:47 +0100 (CET)", "by mail-wr1-x436.google.com with SMTP id h5so1389571wrv.7\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 27 Oct 2020 04:07:47 -0700 (PDT)", "from pi4-davidp.pitowers.org\n\t([2a00:1098:3142:14:1ce1:9965:4328:89c4])\n\tby smtp.gmail.com with ESMTPSA id\n\ti14sm1523230wml.24.2020.10.27.04.07.45\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tTue, 27 Oct 2020 04:07:46 -0700 (PDT)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"DX7aBa4D\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=from:to:cc:subject:date:message-id:in-reply-to:references\n\t:mime-version:content-transfer-encoding;\n\tbh=FvKi2LSp5Fl17Zqyy1HWF/pgLtI043yisvEs+Z9YjR8=;\n\tb=DX7aBa4DJcusaPeDv+k+lx1+aoJcphJIoLlmLwUmdwSyh1XrfltHcv0Gtd8OwzHxh0\n\toVq+quV0X784/SUDzAOz1Xy77BlavLjifGLvCE/dUvSrFj6TRFXhqTxDrB8rIjSKf8ET\n\tMORD/92+uKkBGR/jk+bbHCdxDeRYa8RY//SOOoktz+mMM88iNMRB1sB/4L7JeLL9KIiR\n\t0+jf6nVLxDMtmXULBzh3NREwdhNp/H4FE21bJRKb6odlsvUpuPHhj65vzfOugY8WYKEm\n\tw1+9PQHSUMCspxpfsBD0sP1U2+aypSgddIol5D7nxrEsiFrgcj8rB0/45Z6/CW2HawvX\n\tYj1w==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:mime-version:content-transfer-encoding;\n\tbh=FvKi2LSp5Fl17Zqyy1HWF/pgLtI043yisvEs+Z9YjR8=;\n\tb=uC3dRhD+ddjyyIYRxDWPUzkB4/YuXSyLyw0F+Y3NkNWwBYSy8j0Qr2ou3HWJrOuQQi\n\tv4X3AQfoedaxJh9G0rdK28ctb9NcXli11WBT6x1/2NllzEMand0xZ6x4kaPNZ+GDEwRp\n\tV4rMlMJerj5r93rk7YB+g/h0j2wMvxmvw+OqZxpIoPHvmQfJoPEFkobj4bRkfTigeDLd\n\tmuF6hd/NBjTsNW+OC3kwRRX5xHuM0b6/YD2Dzarx/ftp7QUj++Bvh8n23LWRRNNMIJ2K\n\tjdijzaMtNCbSyWzvvswzpv5YBD1PzMTDZqOx0SJu25276H/zuQGXYbZWrakU7GyvWRYT\n\t3obQ==", "X-Gm-Message-State": "AOAM530uddj92q9W8ftMx3NQXq3f8GHM7VIE63r//UBBx6mfnc1rdi1l\n\tiBmQ/sFuJY7x51q99h7Z18T65dYAvJ5b8A==", "X-Google-Smtp-Source": "ABdhPJyA3M5VT6Y0hvdr5NzDhh0pk6ecUypecJrRyP2X1CTxQF3VLjDTy2XdPjpqplQlWE/q9UMA1w==", "X-Received": "by 2002:a5d:490a:: with SMTP id\n\tx10mr2094471wrq.289.1603796866562; \n\tTue, 27 Oct 2020 04:07:46 -0700 (PDT)", "From": "David Plowman <david.plowman@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Tue, 27 Oct 2020 11:07:38 +0000", "Message-Id": "<20201027110738.15751-1-david.plowman@raspberrypi.com>", "X-Mailer": "git-send-email 2.20.1", "In-Reply-To": "<20201026171908.21463-6-david.plowman@raspberrypi.com>", "References": "<20201026171908.21463-6-david.plowman@raspberrypi.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v7 5/6] libcamera: pipeline: raspberrypi:\n\tImplementation of digital zoom", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "During configure() we update the ScalerCropMaximum to the correct\nvalue for this camera mode and work out the minimum crop size allowed\nby the ISP.\n\nWhenever a new ScalerCrop request is received we check it's valid and\napply it to the ISP V4L2 device. When the IPA returns its metadata to\nus we add the ScalerCrop information, rescaled to sensor native\npixels.\n\nSigned-off-by: David Plowman <david.plowman@raspberrypi.com>\n---\n include/libcamera/ipa/raspberrypi.h | 1 +\n src/ipa/raspberrypi/raspberrypi.cpp | 5 +\n .../pipeline/raspberrypi/raspberrypi.cpp | 93 +++++++++++++++----\n 3 files changed, 82 insertions(+), 17 deletions(-)", "diff": "diff --git a/include/libcamera/ipa/raspberrypi.h b/include/libcamera/ipa/raspberrypi.h\nindex 4ca1528a..2b55dbfc 100644\n--- a/include/libcamera/ipa/raspberrypi.h\n+++ b/include/libcamera/ipa/raspberrypi.h\n@@ -62,6 +62,7 @@ static const ControlInfoMap Controls = {\n \t{ &controls::Saturation, ControlInfo(0.0f, 32.0f) },\n \t{ &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) },\n \t{ &controls::ColourCorrectionMatrix, ControlInfo(-16.0f, 16.0f) },\n+\t{ &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },\n };\n \n } /* namespace RPi */\ndiff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp\nindex 1c255aa2..f338ff8b 100644\n--- a/src/ipa/raspberrypi/raspberrypi.cpp\n+++ b/src/ipa/raspberrypi/raspberrypi.cpp\n@@ -699,6 +699,11 @@ void IPARPi::queueRequest(const ControlList &controls)\n \t\t\tbreak;\n \t\t}\n \n+\t\tcase controls::SCALER_CROP: {\n+\t\t\t/* We do nothing with this, but should avoid the warning below. */\n+\t\t\tbreak;\n+\t\t}\n+\n \t\tdefault:\n \t\t\tLOG(IPARPI, Warning)\n \t\t\t\t<< \"Ctrl \" << controls::controls.at(ctrl.first)->name()\ndiff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\nindex 763451a8..a48013d8 100644\n--- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n+++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n@@ -135,7 +135,7 @@ public:\n \tRPiCameraData(PipelineHandler *pipe)\n \t\t: CameraData(pipe), sensor_(nullptr), state_(State::Stopped),\n \t\t supportsFlips_(false), flipsAlterBayerOrder_(false),\n-\t\t dropFrameCount_(0), ispOutputCount_(0)\n+\t\t updateScalerCrop_(true), dropFrameCount_(0), ispOutputCount_(0)\n \t{\n \t}\n \n@@ -193,6 +193,13 @@ public:\n \tbool flipsAlterBayerOrder_;\n \tBayerFormat::Order nativeBayerOrder_;\n \n+\t/* For handling digital zoom. */\n+\tCameraSensorInfo sensorInfo_;\n+\tRectangle ispCrop_; /* crop in ISP (camera mode) pixels */\n+\tRectangle scalerCrop_; /* crop in sensor native pixels */\n+\tbool updateScalerCrop_;\n+\tSize ispMinCropSize_;\n+\n \tunsigned int dropFrameCount_;\n \n private:\n@@ -677,26 +684,31 @@ int PipelineHandlerRPi::configure(Camera *camera, CameraConfiguration *config)\n \t\treturn ret;\n \t}\n \n-\t/* Adjust aspect ratio by providing crops on the input image. */\n-\tRectangle crop{ 0, 0, sensorFormat.size };\n-\n-\tint ar = maxSize.height * sensorFormat.size.width - maxSize.width * sensorFormat.size.height;\n-\tif (ar > 0)\n-\t\tcrop.width = maxSize.width * sensorFormat.size.height / maxSize.height;\n-\telse if (ar < 0)\n-\t\tcrop.height = maxSize.height * sensorFormat.size.width / maxSize.width;\n+\t/* Figure out the smallest selection the ISP will allow. */\n+\tRectangle testCrop(0, 0, 1, 1);\n+\tdata->isp_[Isp::Input].dev()->setSelection(V4L2_SEL_TGT_CROP, &testCrop);\n+\tdata->ispMinCropSize_ = testCrop.size();\n \n-\tcrop.width &= ~1;\n-\tcrop.height &= ~1;\n+\t/* Adjust aspect ratio by providing crops on the input image. */\n+\tSize size = sensorFormat.size.boundedToAspectRatio(maxSize);\n+\tRectangle crop = size.centeredTo(Rectangle(sensorFormat.size).center());\n+\tdata->ispCrop_ = crop;\n \n-\tcrop.x = (sensorFormat.size.width - crop.width) >> 1;\n-\tcrop.y = (sensorFormat.size.height - crop.height) >> 1;\n \tdata->isp_[Isp::Input].dev()->setSelection(V4L2_SEL_TGT_CROP, &crop);\n \n \tret = data->configureIPA(config);\n \tif (ret)\n \t\tLOG(RPI, Error) << \"Failed to configure the IPA: \" << ret;\n \n+\t/*\n+\t * Update the ScalerCropMaximum to the correct value for this camera mode.\n+\t * For us, it's the same as the \"analogue crop\".\n+\t *\n+\t * \\todo Make this property the ScalerCrop maximum value when dynamic\n+\t * controls are available and set it at validate() time\n+\t */\n+\tdata->properties_.set(properties::ScalerCropMaximum, data->sensorInfo_.analogCrop);\n+\n \treturn ret;\n }\n \n@@ -1154,8 +1166,8 @@ int RPiCameraData::configureIPA(const CameraConfiguration *config)\n \t\tipaConfig.data.push_back(static_cast<unsigned int>(lsTable_.fd()));\n \t}\n \n-\tCameraSensorInfo sensorInfo = {};\n-\tint ret = sensor_->sensorInfo(&sensorInfo);\n+\t/* We store the CameraSensorInfo for digital zoom calculations. */\n+\tint ret = sensor_->sensorInfo(&sensorInfo_);\n \tif (ret) {\n \t\tLOG(RPI, Error) << \"Failed to retrieve camera sensor info\";\n \t\treturn ret;\n@@ -1164,7 +1176,7 @@ int RPiCameraData::configureIPA(const CameraConfiguration *config)\n \t/* Ready the IPA - it must know about the sensor resolution. */\n \tIPAOperationData result;\n \n-\tipa_->configure(sensorInfo, streamConfig, entityControls, ipaConfig,\n+\tipa_->configure(sensorInfo_, streamConfig, entityControls, ipaConfig,\n \t\t\t&result);\n \n \tunsigned int resultIdx = 0;\n@@ -1243,8 +1255,26 @@ void RPiCameraData::queueFrameAction([[maybe_unused]] unsigned int frame,\n \t\tFrameBuffer *buffer = isp_[Isp::Stats].getBuffers().at(bufferId);\n \n \t\thandleStreamBuffer(buffer, &isp_[Isp::Stats]);\n+\n \t\t/* Fill the Request metadata buffer with what the IPA has provided */\n-\t\trequestQueue_.front()->metadata() = std::move(action.controls[0]);\n+\t\tRequest *request = requestQueue_.front();\n+\t\trequest->metadata() = std::move(action.controls[0]);\n+\n+\t\t/*\n+\t\t * Also update the ScalerCrop in the metadata with what we actually\n+\t\t * used. But we must first rescale that from ISP (camera mode) pixels\n+\t\t * back into sensor native pixels.\n+\t\t *\n+\t\t * Sending this information on every frame may be helpful.\n+\t\t */\n+\t\tif (updateScalerCrop_) {\n+\t\t\tupdateScalerCrop_ = false;\n+\t\t\tscalerCrop_ = ispCrop_.scaledBy(sensorInfo_.analogCrop.size(),\n+\t\t\t\t\t\t\tsensorInfo_.outputSize);\n+\t\t\tscalerCrop_.translateBy(sensorInfo_.analogCrop.topLeft());\n+\t\t}\n+\t\trequest->metadata().set(controls::ScalerCrop, scalerCrop_);\n+\n \t\tstate_ = State::IpaComplete;\n \t\tbreak;\n \t}\n@@ -1595,6 +1625,35 @@ void RPiCameraData::tryRunPipeline()\n \t/* Take the first request from the queue and action the IPA. */\n \tRequest *request = requestQueue_.front();\n \n+\tif (request->controls().contains(controls::ScalerCrop)) {\n+\t\tRectangle nativeCrop = request->controls().get<Rectangle>(controls::ScalerCrop);\n+\n+\t\tif (!nativeCrop.width || !nativeCrop.height)\n+\t\t\tnativeCrop = { 0, 0, 1, 1 };\n+\n+\t\t/* Create a version of the crop scaled to ISP (camera mode) pixels. */\n+\t\tRectangle ispCrop = nativeCrop.translatedBy(-sensorInfo_.analogCrop.topLeft());\n+\t\tispCrop.scaleBy(sensorInfo_.outputSize, sensorInfo_.analogCrop.size());\n+\n+\t\t/*\n+\t\t * The crop that we set must be:\n+\t\t * 1. At least as big as ispMinCropSize_, once that's been\n+\t\t * enlarged to the same aspect ratio.\n+\t\t * 2. With the same mid-point, if possible.\n+\t\t * 3. But it can't go outside the sensor area.\n+\t\t */\n+\t\tSize minSize = ispMinCropSize_.expandedToAspectRatio(nativeCrop.size());\n+\t\tSize size = ispCrop.size().expandedTo(minSize);\n+\t\tispCrop = size.centeredTo(ispCrop.center()).enclosedIn(Rectangle(sensorInfo_.outputSize));\n+\n+\t\tif (ispCrop != ispCrop_) {\n+\t\t\tisp_[Isp::Input].dev()->setSelection(V4L2_SEL_TGT_CROP, &ispCrop);\n+\t\t\tispCrop_ = ispCrop;\n+\t\t\t/* queueFrameAction will have to update its scalerCrop_ */\n+\t\t\tupdateScalerCrop_ = true;\n+\t\t}\n+\t}\n+\n \t/*\n \t * Process all the user controls by the IPA. Once this is complete, we\n \t * queue the ISP output buffer listed in the request to start the HW\n", "prefixes": [ "libcamera-devel", "v7", "5/6" ] }