[libcamera-devel,RFC,v2,1/4] pipeline: isp: The software ISP-based pipeline handler

13295 diff mbox series

From: Fan Siyuan <siyuan.fan@foxmail.com>

 Changes in V2:
 -fix the raw and rgb data flow based queue model in pipeline handler
 -move buffer alloc and thread to ISP class
 -Fill metadata information in dstBuffer

Signed-off-by: Fan Siyuan <siyuan.fan@foxmail.com>
---
 src/libcamera/pipeline/isp/isp.cpp | 306 +++++++++++++++++++++++++++++
 1 file changed, 306 insertions(+)
 create mode 100644 src/libcamera/pipeline/isp/isp.cpp

Hi Siyuan,

On Wed, Aug 11, 2021 at 07:12:55AM +0100, Siyuan Fan wrote:
> From: Fan Siyuan <siyuan.fan@foxmail.com>
> 
>  Changes in V2:
>  -fix the raw and rgb data flow based queue model in pipeline handler
>  -move buffer alloc and thread to ISP class
>  -Fill metadata information in dstBuffer
> 
> Signed-off-by: Fan Siyuan <siyuan.fan@foxmail.com>
> ---
>  src/libcamera/pipeline/isp/isp.cpp | 306 +++++++++++++++++++++++++++++
>  1 file changed, 306 insertions(+)
>  create mode 100644 src/libcamera/pipeline/isp/isp.cpp
> 
> diff --git a/src/libcamera/pipeline/isp/isp.cpp b/src/libcamera/pipeline/isp/isp.cpp
> new file mode 100644
> index 00000000..c6b7808c
> --- /dev/null
> +++ b/src/libcamera/pipeline/isp/isp.cpp
> @@ -0,0 +1,306 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Siyuan Fan <siyuan.fan@foxmail.com>
> + *
> + * isp.cpp - The software ISP-based pipeline handler
> + */
> +
> +#include "../../swisp/isp.h"
> +
> +#include <math.h>
> +#include <queue>
> +#include <stdlib.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +#include <libcamera/camera.h>
> +#include <libcamera/control_ids.h>
> +#include <libcamera/controls.h>
> +#include <libcamera/formats.h>
> +
> +#include "libcamera/internal/device_enumerator.h"
> +#include "libcamera/internal/media_device.h"
> +#include "libcamera/internal/pipeline_handler.h"
> +#include "libcamera/internal/v4l2_videodevice.h"
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(ISP)
> +
> +class ISPCameraData : public CameraData
> +{
> +public:
> +       ISPCameraData(PipelineHandler *pipe, MediaDevice *media)
> +               : CameraData(pipe), media_(media), video_(nullptr)
> +       {
> +       }
> +
> +       ~ISPCameraData()
> +       {
> +               delete video_;
> +       }
> +
> +       int init();
> +       void bufferReady(FrameBuffer *buffer);
> +       void ISPCompleted(FrameBuffer *buffer);
> +
> +       Stream stream_;
> +       CPU_ISP isp_;
> +       int width_;
> +       int height_;
> +
> +       std::vector<std::unique_ptr<FrameBuffer>> rawBuffers_;
> +       std::vector<FrameBuffer *> rawQueueBuffers_;
> +       std::vector<FrameBuffer *> rgbQueueBuffers_;

These are queues, so they should be std::queue

Also in english we would say rawBufferQueue.

> +
> +       MediaDevice *media_;
> +       V4L2VideoDevice *video_;
> +};
> +
> +
> +class ISPCameraConfiguration : public CameraConfiguration
> +{
> +public:
> +       ISPCameraConfiguration();
> +
> +       Status validate() override;
> +};
> +
> +class PipelineHandlerISP : public PipelineHandler
> +{
> +public:
> +       PipelineHandlerISP(CameraManager *manager);
> +
> +       CameraConfiguration *generateConfiguration(Camera *camera,
> +                                                   const StreamRoles &roles) override;
> +       int configure(Camera *camera, CameraConfiguration *config) override;
> +
> +       int exportFrameBuffers(Camera *camera, Stream *stream,
> +                               std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
> +
> +       int start(Camera *camera, const ControlList *controls) override;
> +       void stop(Camera *camera) override;
> +
> +       int queueRequestDevice(Camera *camera, Request *request) override;
> +
> +       bool match(DeviceEnumerator *enumerator) override;
> +
> +private:
> +       ISPCameraData *cameraData(const Camera *camera)
> +       {
> +               return static_cast<ISPCameraData *>(
> +                       PipelineHandler::cameraData(camera));
> +       }
> +};
> +
> +ISPCameraConfiguration::ISPCameraConfiguration()
> +       : CameraConfiguration()
> +{
> +}
> +
> +CameraConfiguration::Status ISPCameraConfiguration::validate()
> +{
> +       Status status = Valid;
> +
> +       return status;
> +}
> +
> +PipelineHandlerISP::PipelineHandlerISP(CameraManager *manager)
> +       : PipelineHandler(manager)
> +{
> +}
> +
> +CameraConfiguration *PipelineHandlerISP::generateConfiguration(Camera *camera,
> +                                                               const StreamRoles &roles)
> +{
> +       ISPCameraData *data = cameraData(camera);
> +       CameraConfiguration *config = new ISPCameraConfiguration();
> +
> +       if (roles.empty())
> +               return config;
> +
> +       std::map<V4L2PixelFormat, std::vector<SizeRange>> v4l2Formats =
> +               data->video_->formats();

Similar to this, I think it would be good for the ISP to report what
formats it supports.

> +       std::map<PixelFormat, std::vector<SizeRange>> deviceFormats;
> +       std::transform(v4l2Formats.begin(), v4l2Formats.end(),
> +               std::inserter(deviceFormats, deviceFormats.begin()),
> +               [&](const decltype(v4l2Formats)::value_type &format) {
> +                   return decltype(deviceFormats)::value_type{
> +                       format.first.toPixelFormat(),
> +                       format.second
> +                   };
> +               });
> +
> +       StreamFormats formats(deviceFormats);
> +       StreamConfiguration cfg(formats);
> +
> +       cfg.pixelFormat = formats::RGB888;
> +       cfg.size = { 640, 480 };
> +       cfg.bufferCount = 4;
> +
> +       config->addConfiguration(cfg);
> +
> +       config->validate();
> +
> +       return config;
> +}
> +
> +int PipelineHandlerISP::configure(Camera *camera, CameraConfiguration *config)
> +{
> +       ISPCameraData *data = cameraData(camera);
> +       StreamConfiguration &cfg = config->at(0);
> +
> +       V4L2VideoDevice::Formats fmts = data->video_->formats();
> +       V4L2PixelFormat v4l2Format = fmts.begin()->first;
> +
> +       V4L2DeviceFormat format = {};
> +       format.fourcc = v4l2Format;
> +       format.size = cfg.size;
> +
> +       data->width_ = format.size.width;
> +       data->height_ = format.size.height;
> +
> +       int ret = data->video_->setFormat(&format);
> +       if (ret)
> +               return ret;

The ISP should have a configure() function of some sort as well.

> +
> +       cfg.setStream(&data->stream_);
> +       cfg.stride = format.planes[0].bpl;
> +
> +       return 0;
> +}
> +
> +int PipelineHandlerISP::exportFrameBuffers(Camera *camera, Stream *stream,
> +                                           std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> +{
> +       unsigned int count = stream->configuration().bufferCount;
> +       ISPCameraData *data = cameraData(camera);
> +
> +       count = data->isp_.exportBuffers(buffers, count, data->width_, data->height_);
> +
> +       return count;       
> +
> +}
> +
> +int PipelineHandlerISP::start(Camera *camera, [[maybe_unused]] const ControlList *controls)
> +{
> +       ISPCameraData *data = cameraData(camera);
> +       unsigned int count = data->stream_.configuration().bufferCount;
> +
> +       int ret = data->video_->allocateBuffers(count, &data->rawBuffers_);
> +       if (ret < 0) {
> +              LOG(ISP, Error) << strerror(-ret);
> +              return ret;
> +       }
> +
> +       for (unsigned int i = 0; i < count; i++) {
> +              data->rawQueueBuffers_.push_back(data->rawBuffers_[i].get());
> +       }
> +       
> +
> +       ret = data->video_->streamOn();
> +       if (ret < 0) {
> +               data->video_->releaseBuffers();
> +               return ret;
> +       }
> +
> +       data->isp_.startThreadISP();
> +
> +       return 0;
> +}
> +
> +void PipelineHandlerISP::stop(Camera *camera)
> +{
> +       ISPCameraData *data = cameraData(camera);
> +
> +       if (!(data->rawBuffers_.empty())) {
> +              data->rawBuffers_.clear();
> +       }
> +
> +       data->isp_.stopThreadISP();
> +
> +       data->video_->streamOff();
> +       data->video_->releaseBuffers();
> +}
> +
> +int PipelineHandlerISP::queueRequestDevice(Camera *camera, Request *request)
> +{
> +       ISPCameraData *data = cameraData(camera);
> +       FrameBuffer *rgbBuffer = request->findBuffer(&data->stream_);
> +       data->rgbQueueBuffers_.push_back(rgbBuffer);

This should go after the error check. If there's no buffer then you
should't queue it.

> +       
> +       if (!rgbBuffer) {
> +               LOG(ISP, Error) << "Attempt to queue request with invalid stream";
> +               return -ENOENT;
> +       }
> +

FrameBuffer *buffer = data->rawQueueBuffers_.front();

> +       int ret = data->video_->queueBuffer(data->rawQueueBuffers_[0]);

data->video_->queueBuffer(buffer);

> +       if (ret < 0)
> +               return ret;
> +       FrameBuffer *temp = data->rawQueueBuffers_[0];
> +       data->rawQueueBuffers_.erase(data->rawQueueBuffers_.begin());
> +       data->rawQueueBuffers_.push_back(std::move(temp));

data->video_->rawQueueBuffers_.pop();

Don't requeue the buffer; it'll come back to you in bufferReady.

> +
> +       return 0;
> +}
> +
> +bool PipelineHandlerISP::match(DeviceEnumerator *enumerator)
> +{
> +       DeviceMatch unicam("unicam");
> +
> +       unicam.add("unicam-embedded");
> +       unicam.add("unicam-image");
> +
> +       MediaDevice *unicam_ = acquireMediaDevice(enumerator, unicam);
> +       if (!unicam_) {
> +               LOG(ISP, Debug) << "unicam Device not found";
> +               return false;
> +       }
> +
> +       LOG(ISP, Debug) << "unicam Device Identified";
> +
> +       std::unique_ptr<ISPCameraData> data = std::make_unique<ISPCameraData>(this, unicam_);
> +
> +       if(data->init()) return false;
> +
> +       std::set<Stream *> streams{&data->stream_};
> +       std::shared_ptr<Camera> camera = Camera::create(this, data->video_->deviceName(), streams);
> +       registerCamera(std::move(camera), std::move(data));
> +
> +       return true;
> +}
> +
> +
> +void ISPCameraData::ISPCompleted(FrameBuffer *buffer)
> +{
> +       Request *request = buffer->request();

You need to return the raw buffer to rawQueueBuffers_.

> +
> +       pipe_->completeBuffer(request, buffer);
> +       pipe_->completeRequest(request);
> +
> +}
> +
> +void ISPCameraData::bufferReady(FrameBuffer *buffer)
> +{
> +       LOG(ISP, Debug) << rgbQueueBuffers_[0]->planes()[0].fd.fd();
> +       isp_.invokeMethod(&CPU_ISP::processing, ConnectionTypeQueued, buffer, rgbQueueBuffers_[0], width_, height_);

FrameBuffer *rgbBuf = rgbQueueBuffers_.front();

isp_.invokeMethod(..., buffer, rgbBuf, ...);

> +       rgbQueueBuffers_.erase(rgbQueueBuffers_.begin());
> +       rgbQueueBuffers_.shrink_to_fit();

rgbQueueBuffers_.pop();


Almost there!

Paul

> +       
> +}
> +
> +int ISPCameraData::init()
> +{
> +       video_ = new V4L2VideoDevice(media_->getEntityByName("unicam-image"));
> +       if (video_->open())
> +               return -ENODEV;
> +
> +       video_->bufferReady.connect(this, &ISPCameraData::bufferReady);
> +       isp_.ispCompleted.connect(this, &ISPCameraData::ISPCompleted);
> +
> +       return 0;
> +}
> +
> +REGISTER_PIPELINE_HANDLER(PipelineHandlerISP)
> +
> +} /* namespace libcamera */
> -- 
> 2.20.1
>