[{"id":4334,"web_url":"https://patchwork.libcamera.org/comment/4334/","msgid":"<20200327095551.GC5040@pendragon.ideasonboard.com>","date":"2020-03-27T09:55:51","subject":"Re: [libcamera-devel] [PATCH v2 7/7] libcamera: ipu3: Add support\n\tfor a RAW still capture stream","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas,\n\nThank you for the patch.\n\nOn Thu, Mar 26, 2020 at 11:58:44PM +0100, Niklas Söderlund wrote:\n> Allow the RAW buffer cycling between CIO2 and IMGU to be memory copied\n> to a new FrameBuffer in a new RAW stream. This allows users to capture\n> the raw Bayer format coming from the sensor.\n> \n> As the RAW frame is memory copied queueing requests with the\n> StillCaptureRaw stream might impact performance.\n> \n> Signed-off-by: Niklas Söderlund \n> ---\n> * Changes since v1\n> - Break out translation of mbus code to pixel format to a static map and\n> make use of it in validate() and generateConfiguration()\n> - Move cfg.bufferCount = IPU3_BUFFER_COUNT from adjustStream() to\n> validate()\n> - Added comment and updated an error message.\n> - Added todo to not configure and start the ImgU if only a single raw\n> stream is requested.\n> \n> * Changes from RFC\n> - Add definition for IPU3_MAX_STREAMS.\n> - Deal with all IPU3 Bayer patterns.\n> - Rework size logic.\n> ---\n> src/libcamera/pipeline/ipu3/ipu3.cpp | 111 ++++++++++++++++++++++++---\n> 1 file changed, 101 insertions(+), 10 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> index 0933a03c9f718591..30bc662cc8bfa935 100644\n> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> @@ -33,6 +33,13 @@ LOG_DEFINE_CATEGORY(IPU3)\n> \n> class IPU3CameraData;\n> \n> +static const std::map sensorMbusToPixel = {\n> +\t{ MEDIA_BUS_FMT_SBGGR10_1X10, PixelFormat(DRM_FORMAT_SBGGR10, { IPU3_FORMAT_MOD_PACKED }) },\n> +\t{ MEDIA_BUS_FMT_SGBRG10_1X10, PixelFormat(DRM_FORMAT_SGBRG10, { IPU3_FORMAT_MOD_PACKED }) },\n> +\t{ MEDIA_BUS_FMT_SGRBG10_1X10, PixelFormat(DRM_FORMAT_SGRBG10, { IPU3_FORMAT_MOD_PACKED }) },\n> +\t{ MEDIA_BUS_FMT_SRGGB10_1X10, PixelFormat(DRM_FORMAT_SRGGB10, { IPU3_FORMAT_MOD_PACKED }) },\n> +};\n> +\n> class ImgUDevice\n> {\n> public:\n> @@ -140,11 +147,12 @@ class IPU3Stream : public Stream\n> {\n> public:\n> \tIPU3Stream()\n> -\t\t: active_(false), device_(nullptr)\n> +\t\t: active_(false), raw_(false), device_(nullptr)\n> \t{\n> \t}\n> \n> \tbool active_;\n> +\tbool raw_;\n> \tstd::string name_;\n> \tImgUDevice::ImgUOutput *device_;\n> };\n> @@ -166,6 +174,7 @@ public:\n> \n> \tIPU3Stream outStream_;\n> \tIPU3Stream vfStream_;\n> +\tIPU3Stream rawStream_;\n> };\n> \n> class IPU3CameraConfiguration : public CameraConfiguration\n> @@ -180,6 +189,7 @@ public:\n> \n> private:\n> \tstatic constexpr unsigned int IPU3_BUFFER_COUNT = 4;\n> +\tstatic constexpr unsigned int IPU3_MAX_STREAMS = 3;\n> \n> \tvoid adjustStream(StreamConfiguration &cfg, bool scale);\n> \n> @@ -291,8 +301,6 @@ void IPU3CameraConfiguration::adjustStream(StreamConfiguration &cfg, bool scale)\n> \t\tcfg.size.width &= ~7;\n> \t\tcfg.size.height &= ~3;\n> \t}\n> -\n> -\tcfg.bufferCount = IPU3_BUFFER_COUNT;\n> }\n> \n> CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> @@ -304,8 +312,8 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> \t\treturn Invalid;\n> \n> \t/* Cap the number of entries to the available streams. */\n> -\tif (config_.size() > 2) {\n> -\t\tconfig_.resize(2);\n> +\tif (config_.size() > IPU3_MAX_STREAMS) {\n> +\t\tconfig_.resize(IPU3_MAX_STREAMS);\n> \t\tstatus = Adjusted;\n> \t}\n> \n> @@ -345,6 +353,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> \tstd::set availableStreams = {\n> \t\t&data_->outStream_,\n> \t\t&data_->vfStream_,\n> +\t\t&data_->rawStream_,\n> \t};\n> \n> \tstreams_.clear();\n> @@ -356,7 +365,9 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> \t\tconst Size size = cfg.size;\n> \t\tconst IPU3Stream *stream;\n> \n> -\t\tif (cfg.size == sensorFormat_.size)\n> +\t\tif (cfg.pixelFormat.modifiers().count(IPU3_FORMAT_MOD_PACKED))\n> +\t\t\tstream = &data_->rawStream_;\n> +\t\telse if (cfg.size == sensorFormat_.size)\n> \t\t\tstream = &data_->outStream_;\n> \t\telse\n> \t\t\tstream = &data_->vfStream_;\n> @@ -367,8 +378,20 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> \t\tLOG(IPU3, Debug)\n> \t\t\t<< \"Assigned '\" << stream->name_ << \"' to stream \" << i;\n> \n> -\t\tbool scale = stream == &data_->vfStream_;\n> -\t\tadjustStream(config_[i], scale);\n> +\t\tif (stream->raw_) {\n> +\t\t\tconst auto &itFormat =\n> +\t\t\t\tsensorMbusToPixel.find(sensorFormat_.mbus_code);\n> +\t\t\tif (itFormat == sensorMbusToPixel.end())\n> +\t\t\t\treturn Invalid;\n> +\n> +\t\t\tcfg.pixelFormat = itFormat->second;\n> +\t\t\tcfg.size = sensorFormat_.size;\n> +\t\t} else {\n> +\t\t\tbool scale = stream == &data_->vfStream_;\n> +\t\t\tadjustStream(config_[i], scale);\n> +\t\t}\n> +\n> +\t\tcfg.bufferCount = IPU3_BUFFER_COUNT;\n> \n> \t\tif (cfg.pixelFormat != pixelFormat || cfg.size != size) {\n> \t\t\tLOG(IPU3, Debug)\n> @@ -397,6 +420,7 @@ CameraConfiguration *PipelineHandlerIPU3::generateConfiguration(Camera *camera,\n> \tstd::set streams = {\n> \t\t&data->outStream_,\n> \t\t&data->vfStream_,\n> +\t\t&data->rawStream_,\n> \t};\n> \n> \tconfig = new IPU3CameraConfiguration(camera, data);\n> @@ -438,6 +462,29 @@ CameraConfiguration *PipelineHandlerIPU3::generateConfiguration(Camera *camera,\n> \n> \t\t\tbreak;\n> \n> +\t\tcase StreamRole::StillCaptureRaw: {\n> +\t\t\tif (streams.find(&data->rawStream_) == streams.end()) {\n> +\t\t\t\tLOG(IPU3, Error)\n> +\t\t\t\t\t<< \"No stream available for requested role \"\n> +\t\t\t\t\t<< role;\n> +\t\t\t\tbreak;\n> +\t\t\t}\n> +\n> +\t\t\tstream = &data->rawStream_;\n> +\n> +\t\t\tcfg.size = data->cio2_.sensor_->resolution();\n> +\n> +\t\t\tV4L2SubdeviceFormat sensorFormat =\n> +\t\t\t\tdata->cio2_.sensor_->getFormat({ MEDIA_BUS_FMT_SBGGR10_1X10,\n> +\t\t\t\t\t\t\t\t MEDIA_BUS_FMT_SGBRG10_1X10,\n> +\t\t\t\t\t\t\t\t MEDIA_BUS_FMT_SGRBG10_1X10,\n> +\t\t\t\t\t\t\t\t MEDIA_BUS_FMT_SRGGB10_1X10 },\n> +\t\t\t\t\t\t\t cfg.size);\n> +\t\t\tcfg.pixelFormat =\n> +\t\t\t\tsensorMbusToPixel.at(sensorFormat.mbus_code);\n> +\t\t\tbreak;\n> +\t\t}\n> +\n> \t\tcase StreamRole::Viewfinder:\n> \t\tcase StreamRole::VideoRecording: {\n> \t\t\t/*\n> @@ -535,6 +582,9 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t/*\n> \t * \\todo: Enable links selectively based on the requested streams.\n> \t * As of now, enable all links unconditionally.\n> +\t * \\todo Don't configure the ImgU at all if we only have a single\n> +\t * stream which is for raw capture, in which case no buffers will\n> +\t * never be queued to the ImgU.\n\ns/never/ever/\n\nReviewed-by: Laurent Pinchart \n\n> \t */\n> \tret = data->imgu_->enableLinks(true);\n> \tif (ret)\n> @@ -571,6 +621,13 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> \t\tstream->active_ = true;\n> \t\tcfg.setStream(stream);\n> \n> +\t\t/*\n> +\t\t * The RAW still capture stream just copies buffers from the\n> +\t\t * internal queue and doesn't need any specific configuration.\n> +\t\t */\n> +\t\tif (stream->raw_)\n> +\t\t\tcontinue;\n> +\n> \t\tret = imgu->configureOutput(stream->device_, cfg);\n> \t\tif (ret)\n> \t\t\treturn ret;\n> @@ -621,9 +678,15 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)\n> int PipelineHandlerIPU3::exportFrameBuffers(Camera *camera, Stream *stream,\n> \t\t\t\t\t std::vector> *buffers)\n> {\n> +\tIPU3CameraData *data = cameraData(camera);\n> \tIPU3Stream *ipu3stream = static_cast(stream);\n> -\tV4L2VideoDevice *video = ipu3stream->device_->dev;\n> \tunsigned int count = stream->configuration().bufferCount;\n> +\tV4L2VideoDevice *video;\n> +\n> +\tif (ipu3stream->raw_)\n> +\t\tvideo = data->cio2_.output_;\n> +\telse\n> +\t\tvideo = ipu3stream->device_->dev;\n> \n> \treturn video->exportBuffers(count, buffers);\n> }\n> @@ -737,6 +800,10 @@ int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)\n> \t\tIPU3Stream *stream = static_cast(it.first);\n> \t\tbuffer = it.second;\n> \n> +\t\t/* Skip raw streams, they are copied from the CIO2 buffer. */\n> +\t\tif (stream->raw_)\n> +\t\t\tcontinue;\n> +\n> \t\tint ret = stream->device_->dev->queueBuffer(buffer);\n> \t\tif (ret < 0)\n> \t\t\terror = ret;\n> @@ -831,6 +898,7 @@ int PipelineHandlerIPU3::registerCameras()\n> \t\tstd::set streams = {\n> \t\t\t&data->outStream_,\n> \t\t\t&data->vfStream_,\n> +\t\t\t&data->rawStream_,\n> \t\t};\n> \t\tCIO2Device *cio2 = &data->cio2_;\n> \n> @@ -852,6 +920,8 @@ int PipelineHandlerIPU3::registerCameras()\n> \t\tdata->outStream_.name_ = \"output\";\n> \t\tdata->vfStream_.device_ = &data->imgu_->viewfinder_;\n> \t\tdata->vfStream_.name_ = \"viewfinder\";\n> +\t\tdata->rawStream_.raw_ = true;\n> +\t\tdata->rawStream_.name_ = \"raw\";\n> \n> \t\t/*\n> \t\t * Connect video devices' 'bufferReady' signals to their\n> @@ -941,7 +1011,28 @@ void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)\n> \tif (buffer->metadata().status == FrameMetadata::FrameCancelled)\n> \t\treturn;\n> \n> -\timgu_->input_->queueBuffer(buffer);\n> +\tRequest *request = buffer->request();\n> +\tFrameBuffer *raw = request->findBuffer(&rawStream_);\n> +\n> +\tif (!raw) {\n> +\t\t/* No RAW buffers present, just queue to IMGU. */\n> +\t\timgu_->input_->queueBuffer(buffer);\n> +\t\treturn;\n> +\t}\n> +\n> +\t/* RAW buffers present, special care is needed. */\n> +\tif (request->buffers().size() > 1)\n> +\t\timgu_->input_->queueBuffer(buffer);\n> +\n> +\tif (raw->copyFrom(buffer))\n> +\t\tLOG(IPU3, Debug) << \"Copy of FrameBuffer failed\";\n> +\n> +\tpipe_->completeBuffer(camera_, request, raw);\n> +\n> +\tif (request->buffers().size() == 1) {\n> +\t\tcio2_.putBuffer(buffer);\n> +\t\tpipe_->completeRequest(camera_, request);\n> +\t}\n> }\n> \n> /* -----------------------------------------------------------------------------","headers":{"Return-Path":"","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 459E76040E\n\tfor ;\n\tFri, 27 Mar 2020 10:55:56 +0100 (CET)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A928A2DC;\n\tFri, 27 Mar 2020 10:55:55 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"OaRMZNiE\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1585302955;\n\tbh=w5szMhi1BYPyp4Bp3S+WO5sf1fop9rN2T13MyWV69+E=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=OaRMZNiE4RTN2KCDpFhfs3cf4cDW7GHQ6u1U3H15k/UOUg7tRWNE9oawGMkYPGP1m\n\txzzvRmd8/ahhLlv0WO7hfZZUy7Xxi/cHkTzC8mk0Mxu5vPc6JRQulvS7xzNQYkpQ/0\n\tRP7Qlh5AG+pLEDZGM7DgNE9JlTbSA5op29zL6xHY=","Date":"Fri, 27 Mar 2020 11:55:51 +0200","From":"Laurent Pinchart ","To":"Niklas =?utf-8?q?S=C3=B6derlund?= ","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200327095551.GC5040@pendragon.ideasonboard.com>","References":"<20200326225844.4117712-1-niklas.soderlund@ragnatech.se>\n\t<20200326225844.4117712-8-niklas.soderlund@ragnatech.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200326225844.4117712-8-niklas.soderlund@ragnatech.se>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [PATCH v2 7/7] libcamera: ipu3: Add support\n\tfor a RAW still capture stream","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Fri, 27 Mar 2020 09:55:56 -0000"}}]