[{"id":5100,"web_url":"https://patchwork.libcamera.org/comment/5100/","msgid":"<20200606215910.GI7339@pendragon.ideasonboard.com>","date":"2020-06-06T21:59:10","subject":"Re: [libcamera-devel] [PATCH v2 10/10] libcamera: ipu3: Allow\n\tzero-copy RAW stream capture","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas,\n\nThank you for the patch.\n\nOn Sat, Jun 06, 2020 at 05:04:36PM +0200, Niklas Söderlund wrote:\n> With the refactored CIO2 interface it's now easy to add zero-copy for\n> buffers in the RAW stream. Use the internally allocated buffers inside\n> the CIO2Device if no buffer for the RAW stream is provided by the\n> application, or use the application-provided buffer if any.\n> \n> Signed-off-by: Niklas Söderlund \n> Reviewed-by: Jacopo Mondi \n> ---\n> * Changes since v1\n> - Update comments.\n> - Use Request::findBuffer() instead of own logic.\n> ---\n> src/libcamera/pipeline/ipu3/cio2.cpp | 52 ++++++++++++---------\n> src/libcamera/pipeline/ipu3/cio2.h | 7 ++-\n> src/libcamera/pipeline/ipu3/ipu3.cpp | 69 +++++++++-------------------\n> 3 files changed, 55 insertions(+), 73 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/ipu3/cio2.cpp b/src/libcamera/pipeline/ipu3/cio2.cpp\n> index 9298f10d8d8c07f7..753f20fb615f6748 100644\n> --- a/src/libcamera/pipeline/ipu3/cio2.cpp\n> +++ b/src/libcamera/pipeline/ipu3/cio2.cpp\n> @@ -235,31 +235,16 @@ int CIO2Device::exportBuffers(unsigned int count,\n> \treturn output_->exportBuffers(count, buffers);\n> }\n> \n> -FrameBuffer *CIO2Device::getBuffer()\n> -{\n> -\tif (availableBuffers_.empty()) {\n> -\t\tLOG(IPU3, Error) << \"CIO2 buffer underrun\";\n> -\t\treturn nullptr;\n> -\t}\n> -\n> -\tFrameBuffer *buffer = availableBuffers_.front();\n> -\n> -\tavailableBuffers_.pop();\n> -\n> -\treturn buffer;\n> -}\n> -\n> -void CIO2Device::putBuffer(FrameBuffer *buffer)\n> -{\n> -\tavailableBuffers_.push(buffer);\n> -}\n> -\n> int CIO2Device::start()\n> {\n> -\tint ret = output_->allocateBuffers(CIO2_BUFFER_COUNT, &buffers_);\n> +\tint ret = output_->exportBuffers(CIO2_BUFFER_COUNT, &buffers_);\n> \tif (ret < 0)\n> \t\treturn ret;\n> \n> +\tret = output_->importBuffers(CIO2_BUFFER_COUNT);\n> +\tif (ret)\n> +\t\tLOG(IPU3, Error) << \"Failed to import CIO2 buffers\";\n> +\n> \tfor (std::unique_ptr &buffer : buffers_)\n> \t\tavailableBuffers_.push(buffer.get());\n> \n> @@ -291,11 +276,36 @@ int CIO2Device::stop()\n> \treturn ret;\n> }\n> \n> -int CIO2Device::queueBuffer(FrameBuffer *buffer)\n> +int CIO2Device::queueBuffer(Request *request, FrameBuffer *rawBuffer)\n> {\n> +\tFrameBuffer *buffer = rawBuffer;\n> +\n> +\t/* If no buffer is provided in the request, use an internal one. */\n> +\tif (!buffer) {\n> +\t\tif (availableBuffers_.empty()) {\n> +\t\t\tLOG(IPU3, Error) << \"CIO2 buffer underrun\";\n> +\t\t\treturn -EINVAL;\n> +\t\t}\n> +\n> +\t\tbuffer = availableBuffers_.front();\n> +\t\tavailableBuffers_.pop();\n> +\t}\n> +\n> +\tbuffer->setRequest(request);\n> +\n> \treturn output_->queueBuffer(buffer);\n> }\n> \n> +void CIO2Device::tryReturnBuffer(FrameBuffer *buffer)\n> +{\n> +\tfor (const std::unique_ptr &buf : buffers_) {\n> +\t\tif (buf.get() == buffer) {\n> +\t\t\tavailableBuffers_.push(buffer);\n> +\t\t\tbreak;\n> +\t\t}\n> +\t}\n\nI think you may have missed my comment on v1:\n\nI believe this will be a common need. Should the FrameBuffer class be\nextended with an internal (or external) bool flag ? Or a void *owner (a\nbit of a hack) ? Or something similar but better ? Bonus points if the\nAPI to do so can be hidden from applications.\n\nThis could be done on top, but should be recorded somewhere.\n\n> +}\n> +\n> void CIO2Device::cio2BufferReady(FrameBuffer *buffer)\n> {\n> \tbufferReady.emit(buffer);\n> diff --git a/src/libcamera/pipeline/ipu3/cio2.h b/src/libcamera/pipeline/ipu3/cio2.h\n> index ffb401da6b576556..9bc01b8aa92446b0 100644\n> --- a/src/libcamera/pipeline/ipu3/cio2.h\n> +++ b/src/libcamera/pipeline/ipu3/cio2.h\n> @@ -20,6 +20,7 @@ namespace libcamera {\n> class CameraSensor;\n> class FrameBuffer;\n> class MediaDevice;\n> +class Request;\n> class V4L2DeviceFormat;\n> class V4L2Subdevice;\n> class V4L2VideoDevice;\n> @@ -42,15 +43,13 @@ public:\n> \tint exportBuffers(unsigned int count,\n> \t\t\t std::vector> *buffers);\n> \n> -\tFrameBuffer *getBuffer();\n> -\tvoid putBuffer(FrameBuffer *buffer);\n> -\n> \tint start();\n> \tint stop();\n> \n> \tCameraSensor *sensor() { return sensor_; }\n> \n> -\tint queueBuffer(FrameBuffer *buffer);\n> +\tint queueBuffer(Request *request, FrameBuffer *rawBuffer);\n> +\tvoid tryReturnBuffer(FrameBuffer *buffer);\n> \tSignal bufferReady;\n> \n> private:\n> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> index c95c1bfbce914801..f183ec54daed6f57 100644\n> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> @@ -121,7 +121,6 @@ public:\n> \t}\n> \n> \tvoid imguOutputBufferReady(FrameBuffer *buffer);\n> -\tvoid imguInputBufferReady(FrameBuffer *buffer);\n> \tvoid cio2BufferReady(FrameBuffer *buffer);\n> \n> \tCIO2Device cio2_;\n> @@ -725,25 +724,24 @@ void PipelineHandlerIPU3::stop(Camera *camera)\n> int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)\n> {\n> \tIPU3CameraData *data = cameraData(camera);\n> -\tFrameBuffer *buffer;\n> \tint error = 0;\n> \n> -\t/* Get a CIO2 buffer, associate it with the request and queue it. */\n> -\tbuffer = data->cio2_.getBuffer();\n> -\tif (!buffer)\n> -\t\treturn -EINVAL;\n> -\n> -\tbuffer->setRequest(request);\n> -\tdata->cio2_.queueBuffer(buffer);\n> +\t/*\n> +\t * Queue a buffer on the CIO2, using the raw stream buffer provided in\n> +\t * the request, if any, or a CIO2 internal buffer otherwise.\n> +\t */\n> +\tFrameBuffer *rawBuffer = request->findBuffer(&data->rawStream_);\n> +\terror = data->cio2_.queueBuffer(request, rawBuffer);\n> +\tif (error)\n> +\t\treturn error;\n> \n> +\t/* Queue all buffers from the request aimed for the ImgU. */\n> \tfor (auto it : request->buffers()) {\n> \t\tIPU3Stream *stream = static_cast(it.first);\n> -\t\tbuffer = it.second;\n> -\n> -\t\t/* Skip raw streams, they are copied from the CIO2 buffer. */\n> \t\tif (stream->raw_)\n> \t\t\tcontinue;\n> \n> +\t\tFrameBuffer *buffer = it.second;\n> \t\tint ret = stream->device_->dev->queueBuffer(buffer);\n> \t\tif (ret < 0)\n> \t\t\terror = ret;\n> @@ -873,8 +871,8 @@ int PipelineHandlerIPU3::registerCameras()\n> \t\t */\n> \t\tdata->cio2_.bufferReady.connect(data.get(),\n> \t\t\t\t\t&IPU3CameraData::cio2BufferReady);\n> -\t\tdata->imgu_->input_->bufferReady.connect(data.get(),\n> -\t\t\t\t\t&IPU3CameraData::imguInputBufferReady);\n> +\t\tdata->imgu_->input_->bufferReady.connect(&data->cio2_,\n> +\t\t\t\t\t&CIO2Device::tryReturnBuffer);\n> \t\tdata->imgu_->output_.dev->bufferReady.connect(data.get(),\n> \t\t\t\t\t&IPU3CameraData::imguOutputBufferReady);\n> \t\tdata->imgu_->viewfinder_.dev->bufferReady.connect(data.get(),\n> @@ -903,22 +901,6 @@ int PipelineHandlerIPU3::registerCameras()\n> * Buffer Ready slots\n> */\n> \n> -/**\n> - * \\brief Handle buffers completion at the ImgU input\n> - * \\param[in] buffer The completed buffer\n> - *\n> - * Buffers completed from the ImgU input are immediately queued back to the\n> - * CIO2 unit to continue frame capture.\n> - */\n> -void IPU3CameraData::imguInputBufferReady(FrameBuffer *buffer)\n> -{\n> -\t/* \\todo Handle buffer failures when state is set to BufferError. */\n> -\tif (buffer->metadata().status == FrameMetadata::FrameCancelled)\n> -\t\treturn;\n> -\n> -\tcio2_.putBuffer(buffer);\n> -}\n> -\n> /**\n> * \\brief Handle buffers completion at the ImgU output\n> * \\param[in] buffer The completed buffer\n> @@ -951,27 +933,18 @@ void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)\n> \t\treturn;\n> \n> \tRequest *request = buffer->request();\n> -\tFrameBuffer *raw = request->findBuffer(&rawStream_);\n> \n> -\tif (!raw) {\n> -\t\t/* No RAW buffers present, just queue to IMGU. */\n> -\t\timgu_->input_->queueBuffer(buffer);\n> -\t\treturn;\n> -\t}\n> -\n> -\t/* RAW buffers present, special care is needed. */\n> -\tif (request->buffers().size() > 1)\n> -\t\timgu_->input_->queueBuffer(buffer);\n> -\n> -\tif (raw->copyFrom(buffer))\n> -\t\tLOG(IPU3, Debug) << \"Copy of FrameBuffer failed\";\n> -\n> -\tpipe_->completeBuffer(camera_, request, raw);\n> -\n> -\tif (request->buffers().size() == 1) {\n> -\t\tcio2_.putBuffer(buffer);\n> +\t/*\n> +\t * If the request contains a buffer for the RAW stream only, complete it\n> +\t * now as there's no need for ImgU processing.\n> +\t */\n> +\tif (request->findBuffer(&rawStream_) &&\n> +\t pipe_->completeBuffer(camera_, request, buffer)) {\n> \t\tpipe_->completeRequest(camera_, request);\n> +\t\treturn;\n> \t}\n> +\n> +\timgu_->input_->queueBuffer(buffer);\n> }\n> \n> /* -----------------------------------------------------------------------------","headers":{"Return-Path":"","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 0FBF861167\n\tfor ;\n\tSat, 6 Jun 2020 23:59:30 +0200 (CEST)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 7A98130D;\n\tSat, 6 Jun 2020 23:59:29 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"Ew/7ie7e\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1591480769;\n\tbh=scieyFBjdcgAfPMdiPI4LOG3Qs8Av658W5aQEqOxCaQ=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=Ew/7ie7eIg9uNsZ3tdtyPnlHMzQoKBOykcIvr5087LE0u2NclnKbSR49f7gZLK93g\n\ty9npGZukXLAOdx3R9WfP6vtmcEa29zxM1+NN8D9Cgca3X3sorXk1rxcK/ElSq35T/K\n\tPAtwQuLszThqh7j0r8+0I/xFuUHOB8fOK02BY2m8=","Date":"Sun, 7 Jun 2020 00:59:10 +0300","From":"Laurent Pinchart ","To":"Niklas =?utf-8?q?S=C3=B6derlund?= ","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200606215910.GI7339@pendragon.ideasonboard.com>","References":"<20200606150436.1851700-1-niklas.soderlund@ragnatech.se>\n\t<20200606150436.1851700-11-niklas.soderlund@ragnatech.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200606150436.1851700-11-niklas.soderlund@ragnatech.se>","Subject":"Re: [libcamera-devel] [PATCH v2 10/10] libcamera: ipu3: Allow\n\tzero-copy RAW stream capture","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Sat, 06 Jun 2020 21:59:30 -0000"}}]