[{"id":2863,"web_url":"https://patchwork.libcamera.org/comment/2863/","msgid":"<20191011124309.GD4882@pendragon.ideasonboard.com>","date":"2019-10-11T12:43:09","subject":"Re: [libcamera-devel] [PATCH v6 9/9] libcamera: pipeline: rkisp1:\n\tAttach to an IPA","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas,\n\nThank you for the patch.\n\nOn Fri, Oct 11, 2019 at 05:22:16AM +0200, Niklas Söderlund wrote:\n> Add the plumbing to the pipeline handler to interact with an IPA module.\n> This change makes the usage of an IPA module mandatory for the rkisp1\n> pipeline.\n> \n> The RkISP1 pipeline handler makes use of a timeline component to\n> schedule actions. This might be useful for other pipeline handlers going\n> forward so keep the generic timeline implementation separate to make it\n> easy to break out.\n\nI think there's indeed lots of room for improvement, but that can be\ndone on top of this series. For now this is a good enough proof of\nconcept.\n\nAcked-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n\n> Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>\n> ---\n>  src/libcamera/pipeline/rkisp1/meson.build  |   1 +\n>  src/libcamera/pipeline/rkisp1/rkisp1.cpp   | 548 ++++++++++++++++++++-\n>  src/libcamera/pipeline/rkisp1/timeline.cpp | 227 +++++++++\n>  src/libcamera/pipeline/rkisp1/timeline.h   |  72 +++\n>  4 files changed, 831 insertions(+), 17 deletions(-)\n>  create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp\n>  create mode 100644 src/libcamera/pipeline/rkisp1/timeline.h\n> \n> diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build\n> index f1cc4046b5d064cb..d04fb45223e72fa1 100644\n> --- a/src/libcamera/pipeline/rkisp1/meson.build\n> +++ b/src/libcamera/pipeline/rkisp1/meson.build\n> @@ -1,3 +1,4 @@\n>  libcamera_sources += files([\n>      'rkisp1.cpp',\n> +    'timeline.cpp',\n>  ])\n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index de4ab523d0e4fe36..029d5868d11f5bc9 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -9,32 +9,115 @@\n>  #include <array>\n>  #include <iomanip>\n>  #include <memory>\n> -#include <vector>\n> +#include <queue>\n>  \n>  #include <linux/media-bus-format.h>\n>  \n> +#include <ipa/rkisp1.h>\n> +#include <libcamera/buffer.h>\n>  #include <libcamera/camera.h>\n> +#include <libcamera/control_ids.h>\n>  #include <libcamera/request.h>\n>  #include <libcamera/stream.h>\n>  \n>  #include \"camera_sensor.h\"\n>  #include \"device_enumerator.h\"\n> +#include \"ipa_manager.h\"\n>  #include \"log.h\"\n>  #include \"media_device.h\"\n>  #include \"pipeline_handler.h\"\n> +#include \"timeline.h\"\n>  #include \"utils.h\"\n>  #include \"v4l2_subdevice.h\"\n>  #include \"v4l2_videodevice.h\"\n>  \n> +#define RKISP1_PARAM_BASE 0x100\n> +#define RKISP1_STAT_BASE 0x200\n> +\n>  namespace libcamera {\n>  \n>  LOG_DEFINE_CATEGORY(RkISP1)\n>  \n> +class PipelineHandlerRkISP1;\n> +class RkISP1ActionQueueBuffers;\n> +\n> +enum RkISP1ActionType {\n> +\tSetSensor,\n> +\tSOE,\n> +\tQueueBuffers,\n> +};\n> +\n> +struct RkISP1FrameInfo {\n> +\tunsigned int frame;\n> +\tRequest *request;\n> +\n> +\tBuffer *paramBuffer;\n> +\tBuffer *statBuffer;\n> +\tBuffer *videoBuffer;\n> +\n> +\tbool paramFilled;\n> +\tbool paramDequeued;\n> +\tbool metadataProcessed;\n> +};\n> +\n> +class RkISP1Frames\n> +{\n> +public:\n> +\tRkISP1Frames(PipelineHandler *pipe);\n> +\n> +\tRkISP1FrameInfo *create(unsigned int frame, Request *request, Stream *stream);\n> +\tint destroy(unsigned int frame);\n> +\n> +\tRkISP1FrameInfo *find(unsigned int frame);\n> +\tRkISP1FrameInfo *find(Buffer *buffer);\n> +\tRkISP1FrameInfo *find(Request *request);\n> +\n> +private:\n> +\tPipelineHandlerRkISP1 *pipe_;\n> +\tstd::map<unsigned int, RkISP1FrameInfo *> frameInfo_;\n> +};\n> +\n> +class RkISP1Timeline : public Timeline\n> +{\n> +public:\n> +\tRkISP1Timeline()\n> +\t\t: Timeline()\n> +\t{\n> +\t\tsetDelay(SetSensor, -1, 5);\n> +\t\tsetDelay(SOE, 0, -1);\n> +\t\tsetDelay(QueueBuffers, -1, 10);\n> +\t}\n> +\n> +\tvoid bufferReady(Buffer *buffer)\n> +\t{\n> +\t\t/*\n> +\t\t * Calculate SOE by taking the end of DMA set by the kernel and applying\n> +\t\t * the time offsets provideprovided by the IPA to find the best estimate\n> +\t\t * of SOE.\n> +\t\t */\n> +\n> +\t\tASSERT(frameOffset(SOE) == 0);\n> +\n> +\t\tutils::time_point soe = std::chrono::time_point<utils::clock>()\n> +\t\t\t+ std::chrono::nanoseconds(buffer->timestamp())\n> +\t\t\t+ timeOffset(SOE);\n> +\n> +\t\tnotifyStartOfExposure(buffer->sequence(), soe);\n> +\t}\n> +\n> +\tvoid setDelay(unsigned int type, int frame, int msdelay)\n> +\t{\n> +\t\tutils::duration delay = std::chrono::milliseconds(msdelay);\n> +\t\tsetRawDelay(type, frame, delay);\n> +\t}\n> +};\n> +\n>  class RkISP1CameraData : public CameraData\n>  {\n>  public:\n>  \tRkISP1CameraData(PipelineHandler *pipe)\n> -\t\t: CameraData(pipe), sensor_(nullptr)\n> +\t\t: CameraData(pipe), sensor_(nullptr), frame_(0),\n> +\t\t  frameInfo_(pipe)\n>  \t{\n>  \t}\n>  \n> @@ -43,8 +126,20 @@ public:\n>  \t\tdelete sensor_;\n>  \t}\n>  \n> +\tint loadIPA();\n> +\n>  \tStream stream_;\n>  \tCameraSensor *sensor_;\n> +\tunsigned int frame_;\n> +\tstd::vector<IPABuffer> ipaBuffers_;\n> +\tRkISP1Frames frameInfo_;\n> +\tRkISP1Timeline timeline_;\n> +\n> +private:\n> +\tvoid queueFrameAction(unsigned int frame,\n> +\t\t\t      const IPAOperationData &action);\n> +\n> +\tvoid metadataReady(unsigned int frame, const ControlList &metadata);\n>  };\n>  \n>  class RkISP1CameraConfiguration : public CameraConfiguration\n> @@ -99,18 +194,235 @@ private:\n>  \t\t\tPipelineHandler::cameraData(camera));\n>  \t}\n>  \n> +\tfriend RkISP1ActionQueueBuffers;\n> +\tfriend RkISP1CameraData;\n> +\tfriend RkISP1Frames;\n> +\n>  \tint initLinks();\n>  \tint createCamera(MediaEntity *sensor);\n> +\tvoid tryCompleteRequest(Request *request);\n>  \tvoid bufferReady(Buffer *buffer);\n> +\tvoid paramReady(Buffer *buffer);\n> +\tvoid statReady(Buffer *buffer);\n>  \n>  \tMediaDevice *media_;\n>  \tV4L2Subdevice *dphy_;\n>  \tV4L2Subdevice *isp_;\n>  \tV4L2VideoDevice *video_;\n> +\tV4L2VideoDevice *param_;\n> +\tV4L2VideoDevice *stat_;\n> +\n> +\tBufferPool paramPool_;\n> +\tBufferPool statPool_;\n> +\n> +\tstd::queue<Buffer *> paramBuffers_;\n> +\tstd::queue<Buffer *> statBuffers_;\n>  \n>  \tCamera *activeCamera_;\n>  };\n>  \n> +RkISP1Frames::RkISP1Frames(PipelineHandler *pipe)\n> +\t: pipe_(dynamic_cast<PipelineHandlerRkISP1 *>(pipe))\n> +{\n> +}\n> +\n> +RkISP1FrameInfo *RkISP1Frames::create(unsigned int frame, Request *request, Stream *stream)\n> +{\n> +\tif (pipe_->paramBuffers_.empty()) {\n> +\t\tLOG(RkISP1, Error) << \"Parameters buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\tBuffer *paramBuffer = pipe_->paramBuffers_.front();\n> +\n> +\tif (pipe_->statBuffers_.empty()) {\n> +\t\tLOG(RkISP1, Error) << \"Statisitc buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\tBuffer *statBuffer = pipe_->statBuffers_.front();\n> +\n> +\tBuffer *videoBuffer = request->findBuffer(stream);\n> +\tif (!videoBuffer) {\n> +\t\tLOG(RkISP1, Error)\n> +\t\t\t<< \"Attempt to queue request with invalid stream\";\n> +\t\treturn nullptr;\n> +\t}\n> +\n> +\tpipe_->paramBuffers_.pop();\n> +\tpipe_->statBuffers_.pop();\n> +\n> +\tRkISP1FrameInfo *info = new RkISP1FrameInfo;\n> +\n> +\tinfo->frame = frame;\n> +\tinfo->request = request;\n> +\tinfo->paramBuffer = paramBuffer;\n> +\tinfo->videoBuffer = videoBuffer;\n> +\tinfo->statBuffer = statBuffer;\n> +\tinfo->paramFilled = false;\n> +\tinfo->paramDequeued = false;\n> +\tinfo->metadataProcessed = false;\n> +\n> +\tframeInfo_[frame] = info;\n> +\n> +\treturn info;\n> +}\n> +\n> +int RkISP1Frames::destroy(unsigned int frame)\n> +{\n> +\tRkISP1FrameInfo *info = find(frame);\n> +\tif (!info)\n> +\t\treturn -ENOENT;\n> +\n> +\tpipe_->paramBuffers_.push(info->paramBuffer);\n> +\tpipe_->statBuffers_.push(info->statBuffer);\n> +\n> +\tframeInfo_.erase(info->frame);\n> +\n> +\tdelete info;\n> +\n> +\treturn 0;\n> +}\n> +\n> +RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame)\n> +{\n> +\tauto itInfo = frameInfo_.find(frame);\n> +\n> +\tif (itInfo != frameInfo_.end())\n> +\t\treturn itInfo->second;\n> +\n> +\tLOG(RkISP1, Error) << \"Can't locate info from frame\";\n> +\treturn nullptr;\n> +}\n> +\n> +RkISP1FrameInfo *RkISP1Frames::find(Buffer *buffer)\n> +{\n> +\tfor (auto &itInfo : frameInfo_) {\n> +\t\tRkISP1FrameInfo *info = itInfo.second;\n> +\n> +\t\tif (info->paramBuffer == buffer ||\n> +\t\t    info->statBuffer == buffer ||\n> +\t\t    info->videoBuffer == buffer)\n> +\t\t\treturn info;\n> +\t}\n> +\n> +\tLOG(RkISP1, Error) << \"Can't locate info from buffer\";\n> +\treturn nullptr;\n> +}\n> +\n> +RkISP1FrameInfo *RkISP1Frames::find(Request *request)\n> +{\n> +\tfor (auto &itInfo : frameInfo_) {\n> +\t\tRkISP1FrameInfo *info = itInfo.second;\n> +\n> +\t\tif (info->request == request)\n> +\t\t\treturn info;\n> +\t}\n> +\n> +\tLOG(RkISP1, Error) << \"Can't locate info from request\";\n> +\treturn nullptr;\n> +}\n> +\n> +class RkISP1ActionSetSensor : public FrameAction\n> +{\n> +public:\n> +\tRkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)\n> +\t\t: FrameAction(frame, SetSensor), sensor_(sensor), controls_(controls) {}\n> +\n> +protected:\n> +\tvoid run() override\n> +\t{\n> +\t\tsensor_->setControls(&controls_);\n> +\t}\n> +\n> +private:\n> +\tCameraSensor *sensor_;\n> +\tV4L2ControlList controls_;\n> +};\n> +\n> +class RkISP1ActionQueueBuffers : public FrameAction\n> +{\n> +public:\n> +\tRkISP1ActionQueueBuffers(unsigned int frame, RkISP1CameraData *data,\n> +\t\t\t\t PipelineHandlerRkISP1 *pipe)\n> +\t\t: FrameAction(frame, QueueBuffers), data_(data), pipe_(pipe)\n> +\t{\n> +\t}\n> +\n> +protected:\n> +\tvoid run() override\n> +\t{\n> +\t\tRkISP1FrameInfo *info = data_->frameInfo_.find(frame());\n> +\t\tif (!info)\n> +\t\t\tLOG(RkISP1, Fatal) << \"Frame not known\";\n> +\n> +\t\tif (info->paramFilled)\n> +\t\t\tpipe_->param_->queueBuffer(info->paramBuffer);\n> +\t\telse\n> +\t\t\tLOG(RkISP1, Error)\n> +\t\t\t\t<< \"Parameters not ready on time for frame \"\n> +\t\t\t\t<< frame() << \", ignore parameters.\";\n> +\n> +\t\tpipe_->stat_->queueBuffer(info->statBuffer);\n> +\t\tpipe_->video_->queueBuffer(info->videoBuffer);\n> +\t}\n> +\n> +private:\n> +\tRkISP1CameraData *data_;\n> +\tPipelineHandlerRkISP1 *pipe_;\n> +};\n> +\n> +int RkISP1CameraData::loadIPA()\n> +{\n> +\tipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);\n> +\tif (!ipa_)\n> +\t\treturn -ENOENT;\n> +\n> +\tipa_->queueFrameAction.connect(this,\n> +\t\t\t\t       &RkISP1CameraData::queueFrameAction);\n> +\n> +\treturn 0;\n> +}\n> +\n> +void RkISP1CameraData::queueFrameAction(unsigned int frame,\n> +\t\t\t\t\tconst IPAOperationData &action)\n> +{\n> +\tswitch (action.operation) {\n> +\tcase RKISP1_IPA_ACTION_V4L2_SET: {\n> +\t\tV4L2ControlList controls = action.v4l2controls[0];\n> +\t\ttimeline_.scheduleAction(utils::make_unique<RkISP1ActionSetSensor>(frame,\n> +\t\t\t\t\t\t\t\t\t\t   sensor_,\n> +\t\t\t\t\t\t\t\t\t\t   controls));\n> +\t\tbreak;\n> +\t}\n> +\tcase RKISP1_IPA_ACTION_PARAM_FILLED: {\n> +\t\tRkISP1FrameInfo *info = frameInfo_.find(frame);\n> +\t\tif (info)\n> +\t\t\tinfo->paramFilled = true;\n> +\t\tbreak;\n> +\t}\n> +\tcase RKISP1_IPA_ACTION_METADATA:\n> +\t\tmetadataReady(frame, action.controls[0]);\n> +\t\tbreak;\n> +\tdefault:\n> +\t\tLOG(RkISP1, Error) << \"Unkown action \" << action.operation;\n> +\t\tbreak;\n> +\t}\n> +}\n> +\n> +void RkISP1CameraData::metadataReady(unsigned int frame, const ControlList &metadata)\n> +{\n> +\tPipelineHandlerRkISP1 *pipe =\n> +\t\tstatic_cast<PipelineHandlerRkISP1 *>(pipe_);\n> +\n> +\tRkISP1FrameInfo *info = frameInfo_.find(frame);\n> +\tif (!info)\n> +\t\treturn;\n> +\n> +\tinfo->request->metadata() = metadata;\n> +\tinfo->metadataProcessed = true;\n> +\n> +\tpipe->tryCompleteRequest(info->request);\n> +}\n> +\n>  RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,\n>  \t\t\t\t\t\t     RkISP1CameraData *data)\n>  \t: CameraConfiguration()\n> @@ -202,12 +514,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n>  \n>  PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n>  \t: PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),\n> -\t  video_(nullptr)\n> +\t  video_(nullptr), param_(nullptr), stat_(nullptr)\n>  {\n>  }\n>  \n>  PipelineHandlerRkISP1::~PipelineHandlerRkISP1()\n>  {\n> +\tdelete param_;\n> +\tdelete stat_;\n>  \tdelete video_;\n>  \tdelete isp_;\n>  \tdelete dphy_;\n> @@ -324,6 +638,18 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n>  \t\treturn -EINVAL;\n>  \t}\n>  \n> +\tV4L2DeviceFormat paramFormat = {};\n> +\tparamFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;\n> +\tret = param_->setFormat(&paramFormat);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\tV4L2DeviceFormat statFormat = {};\n> +\tstatFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;\n> +\tret = stat_->setFormat(&statFormat);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n>  \tcfg.setStream(&data->stream_);\n>  \n>  \treturn 0;\n> @@ -332,39 +658,135 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n>  int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,\n>  \t\t\t\t\t   const std::set<Stream *> &streams)\n>  {\n> +\tRkISP1CameraData *data = cameraData(camera);\n>  \tStream *stream = *streams.begin();\n> +\tint ret;\n>  \n>  \tif (stream->memoryType() == InternalMemory)\n> -\t\treturn video_->exportBuffers(&stream->bufferPool());\n> +\t\tret = video_->exportBuffers(&stream->bufferPool());\n>  \telse\n> -\t\treturn video_->importBuffers(&stream->bufferPool());\n> +\t\tret = video_->importBuffers(&stream->bufferPool());\n> +\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\tparamPool_.createBuffers(stream->configuration().bufferCount + 1);\n> +\tret = param_->exportBuffers(&paramPool_);\n> +\tif (ret) {\n> +\t\tvideo_->releaseBuffers();\n> +\t\treturn ret;\n> +\t}\n> +\n> +\tstatPool_.createBuffers(stream->configuration().bufferCount + 1);\n> +\tret = stat_->exportBuffers(&statPool_);\n> +\tif (ret) {\n> +\t\tparam_->releaseBuffers();\n> +\t\tvideo_->releaseBuffers();\n> +\t\treturn ret;\n> +\t}\n> +\n> +\tfor (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {\n> +\t\tdata->ipaBuffers_.push_back({ .id = RKISP1_PARAM_BASE | i,\n> +\t\t\t\t\t      .memory = paramPool_.buffers()[i] });\n> +\t\tparamBuffers_.push(new Buffer(i));\n> +\t}\n> +\n> +\tfor (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {\n> +\t\tdata->ipaBuffers_.push_back({ .id = RKISP1_STAT_BASE | i,\n> +\t\t\t\t\t      .memory = statPool_.buffers()[i] });\n> +\t\tstatBuffers_.push(new Buffer(i));\n> +\t}\n> +\n> +\tdata->ipa_->mapBuffers(data->ipaBuffers_);\n> +\n> +\treturn ret;\n>  }\n>  \n>  int PipelineHandlerRkISP1::freeBuffers(Camera *camera,\n>  \t\t\t\t       const std::set<Stream *> &streams)\n>  {\n> +\tRkISP1CameraData *data = cameraData(camera);\n> +\n> +\twhile (!statBuffers_.empty()) {\n> +\t\tdelete statBuffers_.front();\n> +\t\tstatBuffers_.pop();\n> +\t}\n> +\n> +\twhile (!paramBuffers_.empty()) {\n> +\t\tdelete paramBuffers_.front();\n> +\t\tparamBuffers_.pop();\n> +\t}\n> +\n> +\tstd::vector<unsigned int> ids;\n> +\tfor (IPABuffer &ipabuf : data->ipaBuffers_)\n> +\t\tids.push_back(ipabuf.id);\n> +\n> +\tdata->ipa_->unmapBuffers(ids);\n> +\tdata->ipaBuffers_.clear();\n> +\n> +\tif (param_->releaseBuffers())\n> +\t\tLOG(RkISP1, Error) << \"Failed to release parameters buffers\";\n> +\n> +\tif (stat_->releaseBuffers())\n> +\t\tLOG(RkISP1, Error) << \"Failed to release stat buffers\";\n> +\n>  \tif (video_->releaseBuffers())\n> -\t\tLOG(RkISP1, Error) << \"Failed to release buffers\";\n> +\t\tLOG(RkISP1, Error) << \"Failed to release video buffers\";\n>  \n>  \treturn 0;\n>  }\n>  \n>  int PipelineHandlerRkISP1::start(Camera *camera)\n>  {\n> +\tRkISP1CameraData *data = cameraData(camera);\n>  \tint ret;\n>  \n> +\tdata->frame_ = 0;\n> +\n> +\tret = param_->streamOn();\n> +\tif (ret) {\n> +\t\tLOG(RkISP1, Error)\n> +\t\t\t<< \"Failed to start parameters \" << camera->name();\n> +\t\treturn ret;\n> +\t}\n> +\n> +\tret = stat_->streamOn();\n> +\tif (ret) {\n> +\t\tparam_->streamOff();\n> +\t\tLOG(RkISP1, Error)\n> +\t\t\t<< \"Failed to start statistics \" << camera->name();\n> +\t\treturn ret;\n> +\t}\n> +\n>  \tret = video_->streamOn();\n> -\tif (ret)\n> +\tif (ret) {\n> +\t\tparam_->streamOff();\n> +\t\tstat_->streamOff();\n> +\n>  \t\tLOG(RkISP1, Error)\n>  \t\t\t<< \"Failed to start camera \" << camera->name();\n> +\t}\n>  \n>  \tactiveCamera_ = camera;\n>  \n> +\t/* Inform IPA of stream configuration and sensor controls. */\n> +\tstd::map<unsigned int, IPAStream> streamConfig;\n> +\tstreamConfig[0] = {\n> +\t\t.pixelFormat = data->stream_.configuration().pixelFormat,\n> +\t\t.size = data->stream_.configuration().size,\n> +\t};\n> +\n> +\tstd::map<unsigned int, V4L2ControlInfoMap> entityControls;\n> +\tentityControls[0] = data->sensor_->controls();\n> +\n> +\tdata->ipa_->configure(streamConfig, entityControls);\n> +\n>  \treturn ret;\n>  }\n>  \n>  void PipelineHandlerRkISP1::stop(Camera *camera)\n>  {\n> +\tRkISP1CameraData *data = cameraData(camera);\n>  \tint ret;\n>  \n>  \tret = video_->streamOff();\n> @@ -372,6 +794,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera)\n>  \t\tLOG(RkISP1, Warning)\n>  \t\t\t<< \"Failed to stop camera \" << camera->name();\n>  \n> +\tret = stat_->streamOff();\n> +\tif (ret)\n> +\t\tLOG(RkISP1, Warning)\n> +\t\t\t<< \"Failed to stop statistics \" << camera->name();\n> +\n> +\tret = param_->streamOff();\n> +\tif (ret)\n> +\t\tLOG(RkISP1, Warning)\n> +\t\t\t<< \"Failed to stop parameters \" << camera->name();\n> +\n> +\tdata->timeline_.reset();\n> +\n>  \tactiveCamera_ = nullptr;\n>  }\n>  \n> @@ -380,18 +814,24 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)\n>  \tRkISP1CameraData *data = cameraData(camera);\n>  \tStream *stream = &data->stream_;\n>  \n> -\tBuffer *buffer = request->findBuffer(stream);\n> -\tif (!buffer) {\n> -\t\tLOG(RkISP1, Error)\n> -\t\t\t<< \"Attempt to queue request with invalid stream\";\n> +\tPipelineHandler::queueRequest(camera, request);\n> +\n> +\tRkISP1FrameInfo *info = data->frameInfo_.create(data->frame_, request,\n> +\t\t\t\t\t\t\tstream);\n> +\tif (!info)\n>  \t\treturn -ENOENT;\n> -\t}\n>  \n> -\tint ret = video_->queueBuffer(buffer);\n> -\tif (ret < 0)\n> -\t\treturn ret;\n> +\tIPAOperationData op;\n> +\top.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST;\n> +\top.data = { data->frame_, RKISP1_PARAM_BASE | info->paramBuffer->index() };\n> +\top.controls = { request->controls() };\n> +\tdata->ipa_->processEvent(op);\n>  \n> -\tPipelineHandler::queueRequest(camera, request);\n> +\tdata->timeline_.scheduleAction(utils::make_unique<RkISP1ActionQueueBuffers>(data->frame_,\n> +\t\t\t\t\t\t\t\t\t\t    data,\n> +\t\t\t\t\t\t\t\t\t\t    this));\n> +\n> +\tdata->frame_++;\n>  \n>  \treturn 0;\n>  }\n> @@ -435,11 +875,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)\n>  \tstd::unique_ptr<RkISP1CameraData> data =\n>  \t\tutils::make_unique<RkISP1CameraData>(this);\n>  \n> +\tdata->controlInfo_.emplace(std::piecewise_construct,\n> +\t\t\t\t   std::forward_as_tuple(&controls::AeEnable),\n> +\t\t\t\t   std::forward_as_tuple(false, true));\n> +\n>  \tdata->sensor_ = new CameraSensor(sensor);\n>  \tret = data->sensor_->init();\n>  \tif (ret)\n>  \t\treturn ret;\n>  \n> +\tret = data->loadIPA();\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n>  \tstd::set<Stream *> streams{ &data->stream_ };\n>  \tstd::shared_ptr<Camera> camera =\n>  \t\tCamera::create(this, sensor->name(), streams);\n> @@ -478,7 +926,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)\n>  \tif (video_->open() < 0)\n>  \t\treturn false;\n>  \n> +\tstat_ = V4L2VideoDevice::fromEntityName(media_, \"rkisp1-statistics\");\n> +\tif (stat_->open() < 0)\n> +\t\treturn false;\n> +\n> +\tparam_ = V4L2VideoDevice::fromEntityName(media_, \"rkisp1-input-params\");\n> +\tif (param_->open() < 0)\n> +\t\treturn false;\n> +\n>  \tvideo_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);\n> +\tstat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);\n> +\tparam_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);\n>  \n>  \t/* Configure default links. */\n>  \tif (initLinks() < 0) {\n> @@ -504,13 +962,69 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)\n>   * Buffer Handling\n>   */\n>  \n> +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)\n> +{\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n> +\tRkISP1FrameInfo *info = data->frameInfo_.find(request);\n> +\tif (!info)\n> +\t\treturn;\n> +\n> +\tif (request->hasPendingBuffers())\n> +\t\treturn;\n> +\n> +\tif (!info->metadataProcessed)\n> +\t\treturn;\n> +\n> +\tif (!info->paramDequeued)\n> +\t\treturn;\n> +\n> +\tcompleteRequest(activeCamera_, request);\n> +\n> +\tdata->frameInfo_.destroy(info->frame);\n> +}\n> +\n>  void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)\n>  {\n>  \tASSERT(activeCamera_);\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n>  \tRequest *request = buffer->request();\n>  \n> +\tdata->timeline_.bufferReady(buffer);\n> +\n> +\tif (data->frame_ <= buffer->sequence())\n> +\t\tdata->frame_ = buffer->sequence() + 1;\n> +\n>  \tcompleteBuffer(activeCamera_, request, buffer);\n> -\tcompleteRequest(activeCamera_, request);\n> +\ttryCompleteRequest(request);\n> +}\n> +\n> +void PipelineHandlerRkISP1::paramReady(Buffer *buffer)\n> +{\n> +\tASSERT(activeCamera_);\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n> +\n> +\tRkISP1FrameInfo *info = data->frameInfo_.find(buffer);\n> +\n> +\tinfo->paramDequeued = true;\n> +\ttryCompleteRequest(info->request);\n> +}\n> +\n> +void PipelineHandlerRkISP1::statReady(Buffer *buffer)\n> +{\n> +\tASSERT(activeCamera_);\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n> +\n> +\tRkISP1FrameInfo *info = data->frameInfo_.find(buffer);\n> +\tif (!info)\n> +\t\treturn;\n> +\n> +\tunsigned int frame = info->frame;\n> +\tunsigned int statid = RKISP1_STAT_BASE | info->statBuffer->index();\n> +\n> +\tIPAOperationData op;\n> +\top.operation = RKISP1_IPA_EVENT_SIGNAL_STAT_BUFFER;\n> +\top.data = { frame, statid };\n> +\tdata->ipa_->processEvent(op);\n>  }\n>  \n>  REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);\n> diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp\n> new file mode 100644\n> index 0000000000000000..b98a16689fa994fe\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp\n> @@ -0,0 +1,227 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2019, Google Inc.\n> + *\n> + * timeline.cpp - Timeline for per-frame control\n> + */\n> +\n> +#include \"timeline.h\"\n> +\n> +#include \"log.h\"\n> +\n> +/**\n> + * \\file timeline.h\n> + * \\brief Timeline for per-frame control\n> + */\n> +\n> +namespace libcamera {\n> +\n> +LOG_DEFINE_CATEGORY(Timeline)\n> +\n> +/**\n> + * \\class FrameAction\n> + * \\brief Action that can be schedule on a Timeline\n> + *\n> + * A frame action is an event schedule to be executed on a Timeline. A frame\n> + * action has two primal attributes a frame number and a type.\n> + *\n> + * The frame number describes the frame to which the action is associated. The\n> + * type is a numerical ID which identifies the action within the pipeline and\n> + * IPA protocol.\n> + */\n> +\n> +/**\n> + * \\class Timeline\n> + * \\brief Executor of FrameAction\n> + *\n> + * The timeline has three primary functions:\n> + *\n> + * 1. Keep track of the Start of Exposure (SOE) for every frame processed by\n> + *    the hardware. Using this information it shall keep an up-to-date estimate\n> + *    of the frame interval (time between two consecutive SOE events).\n> + *\n> + *    The estimated frame interval together with recorded SOE events are the\n> + *    foundation for how the timeline schedule FrameAction at specific points\n> + *    in time.\n> + *    \\todo Improve the frame interval estimation algorithm.\n> + *\n> + * 2. Keep track of current delays for different types of actions. The delays\n> + *    for different actions might differ during a capture session. Exposure time\n> + *    effects the over all FPS and different ISP parameters might impacts its\n> + *    processing time.\n> + *\n> + *    The action type delays shall be updated by the IPA in conjunction with\n> + *    how it changes the capture parameters.\n> + *\n> + * 3. Schedule actions on the timeline. This is the process of taking a\n> + *    FrameAction which contains an abstract description of what frame and\n> + *    what type of action it contains and turning that into an time point\n> + *    and make sure the action is executed at that time.\n> + */\n> +\n> +Timeline::Timeline()\n> +\t: frameInterval_(0)\n> +{\n> +\ttimer_.timeout.connect(this, &Timeline::timeout);\n> +}\n> +\n> +/**\n> + * \\brief Reset and stop the timeline\n> + *\n> + * The timeline needs to be reset when the timeline should no longer execute\n> + * actions. A timeline should be reset between two capture sessions to prevent\n> + * the old capture session to effect the second one.\n> + */\n> +void Timeline::reset()\n> +{\n> +\ttimer_.stop();\n> +\n> +\tactions_.clear();\n> +\thistory_.clear();\n> +}\n> +\n> +/**\n> + * \\brief Schedule an action on the timeline\n> + * \\param[in] action FrameAction to schedule\n> + *\n> + * The act of scheduling an action to the timeline is the process of taking\n> + * the properties of the action (type, frame and time offsets) and translating\n> + * that to a time point using the current values for the action type timings\n> + * value recorded in the timeline. If an action is scheduled too late, execute\n> + * it immediately.\n> + */\n> +void Timeline::scheduleAction(std::unique_ptr<FrameAction> action)\n> +{\n> +\tunsigned int lastFrame;\n> +\tutils::time_point lastTime;\n> +\n> +\tif (history_.empty()) {\n> +\t\tlastFrame = 0;\n> +\t\tlastTime = std::chrono::steady_clock::now();\n> +\t} else {\n> +\t\tlastFrame = history_.back().first;\n> +\t\tlastTime = history_.back().second;\n> +\t}\n> +\n> +\t/*\n> +\t * Calculate when the action shall be schedule by first finding out how\n> +\t * many frames in the future the action acts on and then add the actions\n> +\t * frame offset. After the spatial frame offset is found out translate\n> +\t * that to a time point by using the last estimated start of exposure\n> +\t * (SOE) as the fixed offset. Lastly add the action time offset to the\n> +\t * time point.\n> +\t */\n> +\tint frame = action->frame() - lastFrame + frameOffset(action->type());\n> +\tutils::time_point deadline = lastTime + frame * frameInterval_\n> +\t\t+ timeOffset(action->type());\n> +\n> +\tutils::time_point now = std::chrono::steady_clock::now();\n> +\tif (deadline < now) {\n> +\t\tLOG(Timeline, Warning)\n> +\t\t\t<< \"Action scheduled too late \"\n> +\t\t\t<< utils::time_point_to_string(deadline)\n> +\t\t\t<< \", run now \" << utils::time_point_to_string(now);\n> +\t\taction->run();\n> +\t} else {\n> +\t\tactions_.insert({ deadline, std::move(action) });\n> +\t\tupdateDeadline();\n> +\t}\n> +}\n> +\n> +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time)\n> +{\n> +\thistory_.push_back(std::make_pair(frame, time));\n> +\n> +\tif (history_.size() <= HISTORY_DEPTH / 2)\n> +\t\treturn;\n> +\n> +\twhile (history_.size() > HISTORY_DEPTH)\n> +\t\thistory_.pop_front();\n> +\n> +\t/* Update esitmated time between two start of exposures. */\n> +\tutils::duration sumExposures(0);\n> +\tunsigned int numExposures = 0;\n> +\n> +\tutils::time_point lastTime;\n> +\tfor (auto it = history_.begin(); it != history_.end(); it++) {\n> +\t\tif (it != history_.begin()) {\n> +\t\t\tsumExposures += it->second - lastTime;\n> +\t\t\tnumExposures++;\n> +\t\t}\n> +\n> +\t\tlastTime = it->second;\n> +\t}\n> +\n> +\tframeInterval_ = sumExposures;\n> +\tif (numExposures)\n> +\t\tframeInterval_ /= numExposures;\n> +}\n> +\n> +int Timeline::frameOffset(unsigned int type) const\n> +{\n> +\tconst auto it = delays_.find(type);\n> +\tif (it == delays_.end()) {\n> +\t\tLOG(Timeline, Error)\n> +\t\t\t<< \"No frame offset set for action type \" << type;\n> +\t\treturn 0;\n> +\t}\n> +\n> +\treturn it->second.first;\n> +}\n> +\n> +utils::duration Timeline::timeOffset(unsigned int type) const\n> +{\n> +\tconst auto it = delays_.find(type);\n> +\tif (it == delays_.end()) {\n> +\t\tLOG(Timeline, Error)\n> +\t\t\t<< \"No time offset set for action type \" << type;\n> +\t\treturn utils::duration::zero();\n> +\t}\n> +\n> +\treturn it->second.second;\n> +}\n> +\n> +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time)\n> +{\n> +\tdelays_[type] = std::make_pair(frame, time);\n> +}\n> +\n> +void Timeline::updateDeadline()\n> +{\n> +\tif (actions_.empty())\n> +\t\treturn;\n> +\n> +\tconst utils::time_point &deadline = actions_.begin()->first;\n> +\n> +\tif (timer_.isRunning() && deadline >= timer_.deadline())\n> +\t\treturn;\n> +\n> +\tif (deadline <= std::chrono::steady_clock::now()) {\n> +\t\ttimeout(&timer_);\n> +\t\treturn;\n> +\t}\n> +\n> +\ttimer_.start(deadline);\n> +}\n> +\n> +void Timeline::timeout(Timer *timer)\n> +{\n> +\tutils::time_point now = std::chrono::steady_clock::now();\n> +\n> +\tfor (auto it = actions_.begin(); it != actions_.end();) {\n> +\t\tconst utils::time_point &sched = it->first;\n> +\n> +\t\tif (sched > now)\n> +\t\t\tbreak;\n> +\n> +\t\tFrameAction *action = it->second.get();\n> +\n> +\t\taction->run();\n> +\n> +\t\tit = actions_.erase(it);\n> +\t}\n> +\n> +\tupdateDeadline();\n> +}\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/rkisp1/timeline.h b/src/libcamera/pipeline/rkisp1/timeline.h\n> new file mode 100644\n> index 0000000000000000..9d30e4eaf8743d07\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/rkisp1/timeline.h\n> @@ -0,0 +1,72 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2019, Google Inc.\n> + *\n> + * timeline.h - Timeline for per-frame controls\n> + */\n> +#ifndef __LIBCAMERA_TIMELINE_H__\n> +#define __LIBCAMERA_TIMELINE_H__\n> +\n> +#include <list>\n> +#include <map>\n> +\n> +#include <libcamera/timer.h>\n> +\n> +#include \"utils.h\"\n> +\n> +namespace libcamera {\n> +\n> +class FrameAction\n> +{\n> +public:\n> +\tFrameAction(unsigned int frame, unsigned int type)\n> +\t\t: frame_(frame), type_(type) {}\n> +\n> +\tvirtual ~FrameAction() {}\n> +\n> +\tunsigned int frame() const { return frame_; }\n> +\tunsigned int type() const { return type_; }\n> +\n> +\tvirtual void run() = 0;\n> +\n> +private:\n> +\tunsigned int frame_;\n> +\tunsigned int type_;\n> +};\n> +\n> +class Timeline\n> +{\n> +public:\n> +\tTimeline();\n> +\tvirtual ~Timeline() {}\n> +\n> +\tvirtual void reset();\n> +\tvirtual void scheduleAction(std::unique_ptr<FrameAction> action);\n> +\tvirtual void notifyStartOfExposure(unsigned int frame, utils::time_point time);\n> +\n> +\tutils::duration frameInterval() const { return frameInterval_; }\n> +\n> +protected:\n> +\tint frameOffset(unsigned int type) const;\n> +\tutils::duration timeOffset(unsigned int type) const;\n> +\n> +\tvoid setRawDelay(unsigned int type, int frame, utils::duration time);\n> +\n> +\tstd::map<unsigned int, std::pair<int, utils::duration>> delays_;\n> +\n> +private:\n> +\tstatic constexpr unsigned int HISTORY_DEPTH = 10;\n> +\n> +\tvoid timeout(Timer *timer);\n> +\tvoid updateDeadline();\n> +\n> +\tstd::list<std::pair<unsigned int, utils::time_point>> history_;\n> +\tstd::multimap<utils::time_point, std::unique_ptr<FrameAction>> actions_;\n> +\tutils::duration frameInterval_;\n> +\n> +\tTimer timer_;\n> +};\n> +\n> +} /* namespace libcamera */\n> +\n> +#endif /* __LIBCAMERA_TIMELINE_H__ */","headers":{"Return-Path":"<laurent.pinchart@ideasonboard.com>","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 5B02061564\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 11 Oct 2019 14:43:11 +0200 (CEST)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id B294433A;\n\tFri, 11 Oct 2019 14:43:10 +0200 (CEST)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1570797791;\n\tbh=Y1LmyOXutfb9hChJ7xdnQhAyBdDoGxTc+EPVUMjAH9w=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=eN9QH9jCA6HKo5VaATfmu1AO/UT+DjuxLxLhF37aATe7MAZ6VujOfz9KV9QPNNjyG\n\tUjuPo/hUvD7rt66OWnxEW0zhx9soLQdgXGLWzGJi9Wk4GA4lEU2TYHsag72fIC3A3a\n\tc5JcwmXKjmXOW4blwVxqvsNV5cmirxuYzjj6Ybt0=","Date":"Fri, 11 Oct 2019 15:43:09 +0300","From":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","To":"Niklas =?utf-8?q?S=C3=B6derlund?= <niklas.soderlund@ragnatech.se>","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20191011124309.GD4882@pendragon.ideasonboard.com>","References":"<20191011032216.2175173-1-niklas.soderlund@ragnatech.se>\n\t<20191011032216.2175173-10-niklas.soderlund@ragnatech.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20191011032216.2175173-10-niklas.soderlund@ragnatech.se>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [PATCH v6 9/9] libcamera: pipeline: rkisp1:\n\tAttach to an IPA","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","X-List-Received-Date":"Fri, 11 Oct 2019 12:43:11 -0000"}}]