[{"id":24185,"web_url":"https://patchwork.libcamera.org/comment/24185/","msgid":"","date":"2022-07-27T11:10:38","subject":"Re: [libcamera-devel] [PATCH v3 6/9] ipu3: Configure imgu1 when\n\tnecessary","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi Harvey,\n\nOn 6/29/22 16:00, Harvey Yang via libcamera-devel wrote:\n> This patch configure and setup imgu1 when still capture and other\n> streams are enabled. If only still capture is enabled, imgu0 is still\n> being used with IPU3PipeModeVideo (i.e. VideoSnapshot) to ensure 3A is\n> working.\n>\n> Still capture stream will be enabled in the following patches.\n>\n> Signed-off-by: Harvey Yang \n> ---\n> src/libcamera/pipeline/ipu3/ipu3.cpp | 173 +++++++++++++++++++++++----\n> 1 file changed, 149 insertions(+), 24 deletions(-)\n>\n> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> index a13fb881..ec9d14d1 100644\n> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> @@ -55,9 +55,11 @@ public:\n> \n> \tint loadIPA();\n> \n> +\tvoid tryReturnBuffer(FrameBuffer *buffer);\n> \tvoid imguOutputBufferReady(FrameBuffer *buffer);\n> \tvoid cio2BufferReady(FrameBuffer *buffer);\n> \tvoid paramBufferReady(FrameBuffer *buffer);\n> +\tvoid captureParamBufferReady(FrameBuffer *buffer);\n> \tvoid statBufferReady(FrameBuffer *buffer);\n> \tvoid queuePendingRequests();\n> \tvoid cancelPendingRequests();\n> @@ -93,6 +95,8 @@ private:\n> \tvoid paramsBufferReady(unsigned int id);\n> \tvoid setSensorControls(unsigned int id, const ControlList &sensorControls,\n> \t\t\t const ControlList &lensControls);\n> +\n> +\tstd::map bufferReturnCounters;\n\n'_' suffix for private?\n\nThese are not any buffer return counters but cio2 buffer counters. Hence,\n\n\tstd::map cio2BufferReturnCounters_;\n\n    would be more appropriate\n\n> };\n> \n> class IPU3CameraConfiguration : public CameraConfiguration\n> @@ -106,7 +110,8 @@ public:\n> \tStatus validate() override;\n> \n> \tconst StreamConfiguration &cio2Format() const { return cio2Configuration_; }\n> -\tconst ImgUDevice::PipeConfig imguConfig() const { return pipeConfig_; }\n> +\tconst ImgUDevice::PipeConfig imguConfig0() const { return pipeConfig0_; }\n> +\tconst ImgUDevice::PipeConfig imguConfig1() const { return pipeConfig1_; }\n> \n> \t/* Cache the combinedTransform_ that will be applied to the sensor */\n> \tTransform combinedTransform_;\n> @@ -120,7 +125,8 @@ private:\n> \tconst IPU3CameraData *data_;\n> \n> \tStreamConfiguration cio2Configuration_;\n> -\tImgUDevice::PipeConfig pipeConfig_;\n> +\tImgUDevice::PipeConfig pipeConfig0_;\n> +\tImgUDevice::PipeConfig pipeConfig1_;\n> };\n> \n> class PipelineHandlerIPU3 : public PipelineHandler\n> @@ -168,6 +174,8 @@ private:\n> \tMediaDevice *cio2MediaDev_;\n> \tMediaDevice *imguMediaDev_;\n> \n> +\tbool useImgu1_ = false;\n> +\n> \tCamera *inUseCamera_ = nullptr;\n> \n> \tstd::vector ipaBuffers_;\n> @@ -408,8 +416,8 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> \n> \t/* Only compute the ImgU configuration if a YUV stream has been requested. */\n> \tif (yuvCount) {\n> -\t\tpipeConfig_ = data_->imgu0_->calculatePipeConfig(&pipe);\n> -\t\tif (pipeConfig_.isNull()) {\n> +\t\tpipeConfig0_ = data_->imgu0_->calculatePipeConfig(&pipe);\n> +\t\tif (pipeConfig0_.isNull()) {\n> \t\t\tLOG(IPU3, Error) << \"Failed to calculate pipe configuration: \"\n> \t\t\t\t\t << \"unsupported resolutions.\";\n> \t\t\treturn Invalid;\n> @@ -519,8 +527,13 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \tIPU3CameraData *data = cameraData(camera);\n> \tStream *outStream = &data->outStream_;\n> \tStream *vfStream = &data->vfStream_;\n> +\tStream *outCaptureStream = &data->outCaptureStream_;\n> \tCIO2Device *cio2 = &data->cio2_;\n> \tV4L2DeviceFormat outputFormat;\n> +\n> +\tImgUDevice::PipeConfig imguConfig1 = config->imguConfig1();\n> +\tuseImgu1_ = !imguConfig1.isNull();\n> +\n> \tint ret;\n> \n> \t/*\n> @@ -565,6 +578,12 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \tif (ret)\n> \t\treturn ret;\n> \n> +\tif (useImgu1_) {\n> +\t\tret = imgu1_.enableLinks(true);\n> +\t\tif (ret)\n> +\t\t\treturn ret;\n> +\t}\n> +\n> \t/*\n> \t * Pass the requested stream size to the CIO2 unit and get back the\n> \t * adjusted format to be propagated to the ImgU output devices.\n> @@ -607,14 +626,20 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t * stream has been requested: return here to skip the ImgU configuration\n> \t * part.\n> \t */\n> -\tImgUDevice::PipeConfig imguConfig = config->imguConfig();\n> -\tif (imguConfig.isNull())\n> +\tImgUDevice::PipeConfig imguConfig0 = config->imguConfig0();\n> +\tif (imguConfig0.isNull())\n> \t\treturn 0;\n> \n> -\tret = imgu0_.configure(imguConfig, &cio2Format);\n> +\tret = imgu0_.configure(imguConfig0, &cio2Format);\n> \tif (ret)\n> \t\treturn ret;\n> \n> +\tif (useImgu1_) {\n> +\t\tret = imgu1_.configure(imguConfig1, &cio2Format);\n> +\t\tif (ret)\n> +\t\t\treturn ret;\n> +\t}\n> +\n> \t/* Apply the format to the configured streams output devices. */\n> \tStreamConfiguration *mainCfg = nullptr;\n> \tStreamConfiguration *vfCfg = nullptr;\n> @@ -633,6 +658,15 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t\t\tret = imgu0_.configureViewfinder(cfg, &outputFormat);\n> \t\t\tif (ret)\n> \t\t\t\treturn ret;\n> +\t\t} else if (stream == outCaptureStream) {\n> +\t\t\tASSERT(useImgu1_);\n> +\t\t\tret = imgu1_.configureOutput(cfg, &outputFormat);\n> +\t\t\tif (ret)\n> +\t\t\t\treturn ret;\n> +\n> +\t\t\tret = imgu1_.configureViewfinder(cfg, &outputFormat);\n> +\t\t\tif (ret)\n> +\t\t\t\treturn ret;\n> \t\t}\n> \t}\n> \n> @@ -648,22 +682,34 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t}\n> \n> \t/* Apply the \"pipe_mode\" control to the ImgU subdevice. */\n> -\tControlList ctrls(imgu0_.imgu_->controls());\n> +\tControlList ctrls0(imgu0_.imgu_->controls());\n> \t/*\n> \t * Set the ImgU pipe mode to 'Video' unconditionally to have statistics\n> \t * generated.\n> -\t *\n> -\t * \\todo Figure out what the 'Still Capture' mode is meant for, and use\n> -\t * it accordingly.\n> \t */\n> -\tctrls.set(V4L2_CID_IPU3_PIPE_MODE,\n> -\t\t static_cast(IPU3PipeModeVideo));\n> -\tret = imgu0_.imgu_->setControls(&ctrls);\n> +\tctrls0.set(V4L2_CID_IPU3_PIPE_MODE,\n> +\t\t static_cast(IPU3PipeModeVideo));\n> +\tret = imgu0_.imgu_->setControls(&ctrls0);\n> \tif (ret) {\n> \t\tLOG(IPU3, Error) << \"Unable to set pipe_mode control\";\n\n        LOG(IPU3, Error) << \"imgu0: Unable to set pipe_mode control\";\n\n\n> \t\treturn ret;\n> \t}\n> \n> +\tif (useImgu1_) {\n> +\t\tControlList ctrls1(imgu1_.imgu_->controls());\n> +\t\t/*\n> +\t\t * \\todo Figure out what the 'Still Capture' mode is meant for,\n> +\t\t * and use it accordingly.\n> +\t\t */\n> +\t\tctrls1.set(V4L2_CID_IPU3_PIPE_MODE,\n> +\t\t\t static_cast(IPU3PipeModeStillCapture));\n> +\t\tret = imgu1_.imgu_->setControls(&ctrls1);\n> +\t\tif (ret) {\n> +\t\t\tLOG(IPU3, Error) << \"Unable to set pipe_mode control\";\n\n        LOG(IPU3, Error) << \"imgu1: Unable to set pipe_mode control\";\n\n> +\t\t\treturn ret;\n> +\t\t}\n> +\t}\n> +\n> \tipa::ipu3::IPAConfigInfo configInfo;\n> \tconfigInfo.sensorControls = data->cio2_.sensor()->controls();\n> \n> @@ -672,8 +718,8 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t\tconfigInfo.lensControls = lens->controls();\n> \n> \tconfigInfo.sensorInfo = sensorInfo;\n> -\tconfigInfo.bdsOutputSize = config->imguConfig().bds;\n> -\tconfigInfo.iif = config->imguConfig().iif;\n> +\tconfigInfo.bdsOutputSize = config->imguConfig0().bds;\n> +\tconfigInfo.iif = config->imguConfig0().iif;\n> \n> \tret = data->ipa_->configure(configInfo, &data->ipaControls_);\n> \tif (ret) {\n> @@ -793,8 +839,8 @@ int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] const ControlLis\n> \tCIO2Device *cio2 = &data->cio2_;\n> \tint ret;\n> \n> -\timgu0_.input_->bufferReady.connect(&data->cio2_,\n> -\t\t\t\t\t &CIO2Device::tryReturnBuffer);\n> +\timgu0_.input_->bufferReady.connect(data,\n> +\t\t\t\t\t &IPU3CameraData::tryReturnBuffer);\n> \timgu0_.output_->bufferReady.connect(data,\n> \t\t\t\t\t &IPU3CameraData::imguOutputBufferReady);\n> \timgu0_.viewfinder_->bufferReady.connect(data,\n> @@ -803,6 +849,16 @@ int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] const ControlLis\n> \t\t\t\t\t &IPU3CameraData::paramBufferReady);\n> \timgu0_.stat_->bufferReady.connect(data,\n> \t\t\t\t\t &IPU3CameraData::statBufferReady);\n> +\n> +\tif (useImgu1_) {\n> +\t\timgu1_.input_->bufferReady.connect(data,\n> +\t\t\t\t&IPU3CameraData::tryReturnBuffer);\n> +\t\timgu1_.output_->bufferReady.connect(data,\n> +\t\t\t\t&IPU3CameraData::imguOutputBufferReady);\n> +\t\timgu1_.param_->bufferReady.connect(data,\n> +\t\t\t\t&IPU3CameraData::captureParamBufferReady);\n> +\t}\n> +\n> \t/* Disable test pattern mode on the sensor, if any. */\n> \tret = cio2->sensor()->setTestPatternMode(\n> \t\tcontrols::draft::TestPatternModeEnum::TestPatternModeOff);\n> @@ -834,10 +890,17 @@ int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] const ControlLis\n> \tif (ret)\n> \t\tgoto error;\n> \n> +\tif (useImgu1_) {\n> +\t\tret = imgu1_.start();\n> +\t\tif (ret)\n> +\t\t\tgoto error;\n> +\t}\n> +\n> \treturn 0;\n> \n> error:\n> \timgu0_.stop();\n> +\timgu1_.stop();\n> \tcio2->stop();\n> \tdata->ipa_->stop();\n> \tfreeBuffers(camera);\n> @@ -848,6 +911,11 @@ error:\n> \timgu0_.viewfinder_->bufferReady.disconnect();\n> \timgu0_.param_->bufferReady.disconnect();\n> \timgu0_.stat_->bufferReady.disconnect();\n> +\timgu1_.input_->bufferReady.disconnect();\n> +\timgu1_.output_->bufferReady.disconnect();\n> +\timgu1_.param_->bufferReady.disconnect();\n> +\n> +\tuseImgu1_ = false;\n> \tinUseCamera_ = nullptr;\n> \n> \treturn ret;\n> @@ -863,6 +931,7 @@ void PipelineHandlerIPU3::stopDevice(Camera *camera)\n> \tdata->ipa_->stop();\n> \n> \tret |= imgu0_.stop();\n> +\tret |= imgu1_.stop();\n> \tret |= data->cio2_.stop();\n> \tif (ret)\n> \t\tLOG(IPU3, Warning) << \"Failed to stop camera \" << camera->id();\n> @@ -874,7 +943,11 @@ void PipelineHandlerIPU3::stopDevice(Camera *camera)\n> \tdata->imgu0_->viewfinder_->bufferReady.disconnect();\n> \tdata->imgu0_->param_->bufferReady.disconnect();\n> \tdata->imgu0_->stat_->bufferReady.disconnect();\n> +\tdata->imgu1_->input_->bufferReady.disconnect();\n> +\tdata->imgu1_->output_->bufferReady.disconnect();\n> +\tdata->imgu1_->param_->bufferReady.disconnect();\n> \n> +\tuseImgu1_ = false;\n\n\nAs mentioned in the previous patches, it would be better abstracted \nunder IPU3CameraData helper function.\n\n> \tinUseCamera_ = nullptr;\n> }\n> \n> @@ -1320,22 +1393,45 @@ void IPU3CameraData::paramsBufferReady(unsigned int id)\n> \tif (!info)\n> \t\treturn;\n> \n> +\tbool hasYuv = false;\n> +\tbool hasCapture = false;\n> \t/* Queue all buffers from the request aimed for the ImgU. */\n> \tfor (auto it : info->request->buffers()) {\n> \t\tconst Stream *stream = it.first;\n> \t\tFrameBuffer *outbuffer = it.second;\n> \n> -\t\tif (stream == &outStream_)\n> +\t\tif (stream == &outStream_) {\n> +\t\t\thasYuv = true;\n> \t\t\timgu0_->output_->queueBuffer(outbuffer);\n> -\t\telse if (stream == &vfStream_)\n> +\t\t} else if (stream == &vfStream_) {\n> +\t\t\thasYuv = true;\n> \t\t\timgu0_->viewfinder_->queueBuffer(outbuffer);\n> +\t\t} else if (stream == &outCaptureStream_) {\n> +\t\t\thasCapture = true;\n> +\n> +\t\t\timgu1_->output_->queueBuffer(outbuffer);\n> +\t\t}\n\n\nYes, this was what I was thinking a while ago.\n\nI wonder this fundamental question that if queuing buffers  both to \noutput of imgu0 and output of imgu1 makes sense in any case?\n\nOr is it the case that we need to queue buffers to output get imgu0 \nstreaming, otherwise it'll stall?\n\n> \t}\n> \n> -\timgu0_->param_->queueBuffer(info->paramBuffer);\n> -\timgu0_->stat_->queueBuffer(info->statBuffer);\n> -\timgu0_->input_->queueBuffer(info->rawBuffer);\n> +\tif (hasYuv) {\n> +\t\tbufferReturnCounters[info->rawBuffer] += 1;\n> \n> -\tinfo->captureParamDequeued = true;\n> +\t\timgu0_->param_->queueBuffer(info->paramBuffer);\n> +\t\timgu0_->stat_->queueBuffer(info->statBuffer);\n> +\t\timgu0_->input_->queueBuffer(info->rawBuffer);\n> +\t} else {\n> +\t\tinfo->paramDequeued = true;\n> +\t\tinfo->metadataProcessed = true;\n> +\t}\n> +\n> +\tif (hasCapture) {\n> +\t\tbufferReturnCounters[info->rawBuffer] += 1;\n> +\n> +\t\timgu1_->param_->queueBuffer(info->captureParamBuffer);\n> +\t\timgu1_->input_->queueBuffer(info->rawBuffer);\n> +\t} else {\n> +\t\tinfo->captureParamDequeued = true;\n> +\t}\n> }\n> \n> void IPU3CameraData::metadataReady(unsigned int id, const ControlList &metadata)\n> @@ -1356,6 +1452,15 @@ void IPU3CameraData::metadataReady(unsigned int id, const ControlList &metadata)\n> * Buffer Ready slots\n> */\n> \n> +void IPU3CameraData::tryReturnBuffer(FrameBuffer *buffer)\n> +{\n> +\tif (--bufferReturnCounters[buffer] > 0)\n> +\t\treturn;\n> +\n> +\tbufferReturnCounters.erase(buffer);\n> +\tcio2_.tryReturnBuffer(buffer);\n> +}\n> +\n> /**\n> * \\brief Handle buffers completion at the ImgU output\n> * \\param[in] buffer The completed buffer\n> @@ -1448,6 +1553,26 @@ void IPU3CameraData::paramBufferReady(FrameBuffer *buffer)\n> \t\tpipe()->completeRequest(request);\n> }\n> \n> +void IPU3CameraData::captureParamBufferReady(FrameBuffer *buffer)\n> +{\n> +\tIPU3Frames::Info *info = frameInfos_.find(buffer);\n> +\tif (!info)\n> +\t\treturn;\n> +\n> +\tinfo->captureParamDequeued = true;\n> +\n> +\t/*\n> +\t * tryComplete() will delete info if it completes the IPU3Frame.\n> +\t * In that event, we must have obtained the Request before hand.\n> +\t *\n> +\t * \\todo Improve the FrameInfo API to avoid this type of issue\n> +\t */\n> +\tRequest *request = info->request;\n> +\n> +\tif (frameInfos_.tryComplete(info))\n> +\t\tpipe()->completeRequest(request);\n> +}\n> +\n> void IPU3CameraData::statBufferReady(FrameBuffer *buffer)\n> {\n> \tIPU3Frames::Info *info = frameInfos_.find(buffer);","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 901C2C3275\n\tfor ;\n\tWed, 27 Jul 2022 11:10:48 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C898363313;\n\tWed, 27 Jul 2022 13:10:47 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 8300C63309\n\tfor ;\n\tWed, 27 Jul 2022 13:10:45 +0200 (CEST)","from [IPV6:2401:4900:1f3e:f7a:bc8f:12ed:b45f:c35d] (unknown\n\t[IPv6:2401:4900:1f3e:f7a:bc8f:12ed:b45f:c35d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A95BB835;\n\tWed, 27 Jul 2022 13:10:43 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1658920247;\n\tbh=iwf78gvLNNN1qGNzUshHym2Z2ACHhbp1I1f8FKz0N3Y=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=t9RkywF8BdsxRXOJIEhwTf246f5fZBB0lrRsvPAEw2ix00g0bNzzHNQXO4d83e5nU\n\tjOhKuIK8J4A9csaFpGHfbPZshTpewaAeD0Z1R72UNcAX6YWF9Iw9StmfM7lsxgDH5S\n\tFbKrh7txSYT5oSqNteO60RwTWMsyEyHqqituyQwBNwtkM8tt9hJu0qwsOG/QPl6YgC\n\tmyhoRAt4FWQx/f2AQiV2Z6Oq1eLHDNJ39wv6vWRgJr8YowSbbMs8Cz+bNdZr81lEtH\n\tA1ON1/rVYaU8iqW3+PsE77GtqmZy7fIB/hznZosybpgR7j+D5wirESh+89Xj9m+vKC\n\t7poSNF/4hJdAA==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1658920245;\n\tbh=iwf78gvLNNN1qGNzUshHym2Z2ACHhbp1I1f8FKz0N3Y=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=r0JzXTCms2ue5iEsQC0aGtqSM6S/OijH8i8IEtDExEcoHNdaCvVn+H6T5aeZlJwP4\n\tdRcroxXHWy8S1slQKq611zZZVl/QkBHbhYtihLdeCsqLfNqNf5GDCGbwuzB1z2AOww\n\tuuHoXiv76HZmJpoTL4NyWHcV77dX+nsZJTpfDAIg="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"r0JzXTCm\"; dkim-atps=neutral","Message-ID":"","Date":"Wed, 27 Jul 2022 16:40:38 +0530","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101\n\tThunderbird/91.4.1","Content-Language":"en-US","To":"Harvey Yang ,\n\tlibcamera-devel@lists.libcamera.org","References":"<20220629103018.4025635-1-chenghaoyang@google.com>\n\t<20220629103018.4025635-7-chenghaoyang@google.com>","In-Reply-To":"<20220629103018.4025635-7-chenghaoyang@google.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"8bit","Subject":"Re: [libcamera-devel] [PATCH v3 6/9] ipu3: Configure imgu1 when\n\tnecessary","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Umang Jain via libcamera-devel ","Reply-To":"Umang Jain ","Cc":"Harvey Yang ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]