From patchwork Thu Oct 3 17:49:41 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2093 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6069660BE9 for ; Thu, 3 Oct 2019 19:50:45 +0200 (CEST) X-Halon-ID: 3573d54b-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 3573d54b-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:55 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:41 +0200 Message-Id: <20191003174941.1296988-12-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 11/11] libcamera: pipeline: rkisp1: Attach to an IPA X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:45 -0000 Add the plumbing to the pipeline handler to interact with an IPA module. This change makes the usage of an IPA module mandatory for the rkisp1 pipeline. Signed-off-by: Niklas Söderlund --- src/libcamera/pipeline/rkisp1/meson.build | 1 + src/libcamera/pipeline/rkisp1/rkisp1.cpp | 326 ++++++++++++++++++++- src/libcamera/pipeline/rkisp1/rkisp1.h | 78 +++++ src/libcamera/pipeline/rkisp1/timeline.cpp | 56 ++++ 4 files changed, 448 insertions(+), 13 deletions(-) create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1.h create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build index f1cc4046b5d064cb..d04fb45223e72fa1 100644 --- a/src/libcamera/pipeline/rkisp1/meson.build +++ b/src/libcamera/pipeline/rkisp1/meson.build @@ -1,3 +1,4 @@ libcamera_sources += files([ 'rkisp1.cpp', + 'timeline.cpp', ]) diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index de4ab523d0e4fe36..ade8c8823a0ad99d 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -5,20 +5,22 @@ * rkisp1.cpp - Pipeline handler for Rockchip ISP1 */ +#include "rkisp1.h" + #include #include #include #include -#include #include #include +#include #include #include -#include "camera_sensor.h" #include "device_enumerator.h" +#include "ipa_manager.h" #include "log.h" #include "media_device.h" #include "pipeline_handler.h" @@ -34,7 +36,7 @@ class RkISP1CameraData : public CameraData { public: RkISP1CameraData(PipelineHandler *pipe) - : CameraData(pipe), sensor_(nullptr) + : CameraData(pipe), sensor_(nullptr), frame_(0) { } @@ -43,8 +45,21 @@ public: delete sensor_; } + int loadIPA(); + Stream stream_; CameraSensor *sensor_; + unsigned int frame_; + std::vector ipaBuffers_; + std::map frameInfo_; + RkISP1Timeline timeline_; + +private: + void queueFrameAction(const IPAOperationData &action); + + void queueBuffer(unsigned int frame, unsigned int type, + unsigned int id); + void metaDataReady(unsigned int frame, unsigned int aeState); }; class RkISP1CameraConfiguration : public CameraConfiguration @@ -99,18 +114,114 @@ private: PipelineHandler::cameraData(camera)); } + friend RkISP1CameraData; + int initLinks(); int createCamera(MediaEntity *sensor); + void tryCompleteRequest(Request *request); void bufferReady(Buffer *buffer); + void paramReady(Buffer *buffer); + void statReady(Buffer *buffer); MediaDevice *media_; V4L2Subdevice *dphy_; V4L2Subdevice *isp_; V4L2VideoDevice *video_; + V4L2VideoDevice *param_; + V4L2VideoDevice *stat_; + + BufferPool paramPool_; + BufferPool statPool_; + + std::map paramBuffers_; + std::map statBuffers_; Camera *activeCamera_; }; +int RkISP1CameraData::loadIPA() +{ + ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->queueFrameAction.connect(this, + &RkISP1CameraData::queueFrameAction); + + return 0; +} + +void RkISP1CameraData::queueFrameAction(const IPAOperationData &action) +{ + switch (action.operation) { + case RKISP1_IPA_ACTION_V4L2_SET: { + unsigned int frame = action.data[0]; + V4L2ControlList controls; + for (unsigned int i = 1; i < action.data.size(); i += 2) + controls.add(action.data[i], action.data[i + 1]); + timeline_.scheduleAction(new RkISP1ActionSetSensor(frame, sensor_, controls)); + break; + } + case RKISP1_IPA_ACTION_QUEUE_BUFFER: { + unsigned int frame = action.data[0]; + unsigned int type = action.data[1]; + unsigned int id = action.data[2]; + queueBuffer(frame, type, id); + break; + } + case RKISP1_IPA_ACTION_META_DATA: { + unsigned int frame = action.data[0]; + unsigned aeState = action.data[1]; + metaDataReady(frame, aeState); + break; + } + default: + LOG(RkISP1, Error) << "Unkown action " << action.operation; + break; + } +} + +void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type, + unsigned int id) +{ + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + RkISP1ActionType acttype; + V4L2VideoDevice *device; + Buffer *buffer; + switch (type) { + case RKISP1_BUFFER_PARAM: + acttype = QueueParameters; + device = pipe->param_; + buffer = pipe->paramBuffers_[id]; + break; + case RKISP1_BUFFER_STAT: + acttype = QueueStatistics; + device = pipe->stat_; + buffer = pipe->statBuffers_[id]; + break; + default: + LOG(RkISP1, Error) << "Unkown IPA buffer type " << type; + return; + } + + timeline_.scheduleAction(new RkISP1ActionQueueBuffer(frame, acttype, + device, buffer)); +} + +void RkISP1CameraData::metaDataReady(unsigned int frame, unsigned int aeStatus) +{ + Request *request = frameInfo_[frame]; + PipelineHandlerRkISP1 *pipe = + static_cast(pipe_); + + if (aeStatus) + request->metaData().set(controls::AeLocked, aeStatus == 2); + + pipe->tryCompleteRequest(request); +} + RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera, RkISP1CameraData *data) : CameraConfiguration() @@ -202,12 +313,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) : PipelineHandler(manager), dphy_(nullptr), isp_(nullptr), - video_(nullptr) + video_(nullptr), param_(nullptr), stat_(nullptr) { } PipelineHandlerRkISP1::~PipelineHandlerRkISP1() { + delete param_; + delete stat_; delete video_; delete isp_; delete dphy_; @@ -317,6 +430,20 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) if (ret) return ret; + V4L2DeviceFormat paramFormat = {}; + paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS; + + ret = param_->setFormat(¶mFormat); + if (ret) + return ret; + + V4L2DeviceFormat statFormat = {}; + statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A; + + ret = stat_->setFormat(&statFormat); + if (ret) + return ret; + if (outputFormat.size != cfg.size || outputFormat.fourcc != cfg.pixelFormat) { LOG(RkISP1, Error) @@ -332,39 +459,133 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) int PipelineHandlerRkISP1::allocateBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); Stream *stream = *streams.begin(); + int ret; if (stream->memoryType() == InternalMemory) - return video_->exportBuffers(&stream->bufferPool()); + ret = video_->exportBuffers(&stream->bufferPool()); else - return video_->importBuffers(&stream->bufferPool()); + ret = video_->importBuffers(&stream->bufferPool()); + + if (ret) + return ret; + + paramPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = param_->exportBuffers(¶mPool_); + if (ret) { + video_->releaseBuffers(); + return ret; + } + + statPool_.createBuffers(stream->configuration().bufferCount + 1); + ret = stat_->exportBuffers(&statPool_); + if (ret) { + param_->releaseBuffers(); + video_->releaseBuffers(); + return ret; + } + + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) { + paramBuffers_[i] = new Buffer(i); + statBuffers_[i] = new Buffer(i); + + data->ipaBuffers_.push_back({ + .id = i, + .type = RKISP1_BUFFER_PARAM, + .buffer = paramPool_.buffers()[i], + }); + data->ipaBuffers_.push_back({ + .id = i, + .type = RKISP1_BUFFER_STAT, + .buffer = statPool_.buffers()[i], + }); + } + + data->ipa_->mapBuffers(data->ipaBuffers_); + + return ret; } int PipelineHandlerRkISP1::freeBuffers(Camera *camera, const std::set &streams) { + RkISP1CameraData *data = cameraData(camera); + + data->ipa_->unmapBuffers(data->ipaBuffers_); + data->ipaBuffers_.clear(); + + for (auto it : paramBuffers_) + delete it.second; + + paramBuffers_.clear(); + + for (auto it : statBuffers_) + delete it.second; + + statBuffers_.clear(); + + if (param_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release parameters buffers"; + + if (stat_->releaseBuffers()) + LOG(RkISP1, Error) << "Failed to release stat buffers"; + if (video_->releaseBuffers()) - LOG(RkISP1, Error) << "Failed to release buffers"; + LOG(RkISP1, Error) << "Failed to release video buffers"; return 0; } int PipelineHandlerRkISP1::start(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; + ret = param_->streamOn(); + if (ret) { + LOG(RkISP1, Error) + << "Failed to start parameters " << camera->name(); + return ret; + } + + ret = stat_->streamOn(); + if (ret) { + param_->streamOff(); + LOG(RkISP1, Error) + << "Failed to start statistics " << camera->name(); + return ret; + } + ret = video_->streamOn(); - if (ret) + if (ret) { + param_->streamOff(); + stat_->streamOff(); + LOG(RkISP1, Error) << "Failed to start camera " << camera->name(); + } activeCamera_ = camera; + /* Inform IPA of stream configuration and sensor controls. */ + std::map streamConfig; + streamConfig[0] = { + .pixelFormat = data->stream_.configuration().pixelFormat, + .size = data->stream_.configuration().size, + }; + + std::map entityControls; + entityControls[0] = data->sensor_->controls(); + + data->ipa_->configure(streamConfig, entityControls); + return ret; } void PipelineHandlerRkISP1::stop(Camera *camera) { + RkISP1CameraData *data = cameraData(camera); int ret; ret = video_->streamOff(); @@ -372,6 +593,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera) LOG(RkISP1, Warning) << "Failed to stop camera " << camera->name(); + ret = stat_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop statistics " << camera->name(); + + ret = param_->streamOff(); + if (ret) + LOG(RkISP1, Warning) + << "Failed to stop parameters " << camera->name(); + + data->timeline_.reset(); + activeCamera_ = nullptr; } @@ -387,12 +620,22 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request) return -ENOENT; } - int ret = video_->queueBuffer(buffer); - if (ret < 0) - return ret; - PipelineHandler::queueRequest(camera, request); + data->frameInfo_[data->frame_] = request; + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST; + op.data = { data->frame_ }; + op.controls = { request->controls() }; + data->ipa_->processEvent(op); + + data->timeline_.scheduleAction(new RkISP1ActionQueueBuffer(data->frame_, + QueueVideo, + video_, + buffer)); + data->frame_++; + return 0; } @@ -435,11 +678,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor) std::unique_ptr data = utils::make_unique(this); + data->controlInfo_.emplace(std::piecewise_construct, + std::forward_as_tuple(&controls::AeEnable), + std::forward_as_tuple(false, true)); + data->sensor_ = new CameraSensor(sensor); ret = data->sensor_->init(); if (ret) return ret; + ret = data->loadIPA(); + if (ret) + return ret; + std::set streams{ &data->stream_ }; std::shared_ptr camera = Camera::create(this, sensor->name(), streams); @@ -478,7 +729,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) if (video_->open() < 0) return false; + stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics"); + if (stat_->open() < 0) + return false; + + param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params"); + if (param_->open() < 0) + return false; + video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady); + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); /* Configure default links. */ if (initLinks() < 0) { @@ -504,13 +765,52 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) * Buffer Handling */ +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request) +{ + if (request->hasPendingBuffers()) + return; + + if (request->metaData().empty()) + return; + + completeRequest(activeCamera_, request); +} + void PipelineHandlerRkISP1::bufferReady(Buffer *buffer) { ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); Request *request = buffer->request(); + data->timeline_.bufferReady(buffer); + + if (data->frame_ <= buffer->sequence()) + data->frame_ = buffer->sequence() + 1; + completeBuffer(activeCamera_, request, buffer); - completeRequest(activeCamera_, request); + tryCompleteRequest(request); +} + +void PipelineHandlerRkISP1::paramReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_PARAM, buffer->index() }; + data->ipa_->processEvent(op); +} + +void PipelineHandlerRkISP1::statReady(Buffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + + IPAOperationData op; + op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER; + op.data = { RKISP1_BUFFER_STAT, buffer->index() }; + data->ipa_->processEvent(op); } REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1); diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h new file mode 100644 index 0000000000000000..1972c8bbc1286e0e --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/rkisp1.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * rkisp1.h - Pipeline handler for Rockchip ISP1 + */ +#ifndef __LIBCAMERA_RKISP1_H__ +#define __LIBCAMERA_RKISP1_H__ + +#include +#include + +#include "camera_sensor.h" +#include "timeline.h" +#include "v4l2_videodevice.h" + +namespace libcamera { + +enum RkISP1ActionType { + SetSensor, + SOE, + QueueVideo, + QueueParameters, + QueueStatistics, +}; + +class RkISP1ActionSetSensor : public FrameAction +{ +public: + RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls) + : FrameAction(SetSensor, frame), sensor_(sensor), controls_(controls) {} + +protected: + void run() override; + +private: + CameraSensor *sensor_; + V4L2ControlList controls_; +}; + +class RkISP1ActionQueueBuffer : public FrameAction +{ +public: + RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type, + V4L2VideoDevice *device, Buffer *buffer) + : FrameAction(type, frame), device_(device), buffer_(buffer) + { + } + +protected: + void run() override; + +private: + V4L2VideoDevice *device_; + Buffer *buffer_; +}; + +class RkISP1Timeline : public Timeline +{ +public: + RkISP1Timeline() + : Timeline() + { + setDelay(SetSensor, -1, 5); + setDelay(SOE, 0, -1); + setDelay(QueueVideo, -1, 10); + setDelay(QueueParameters, -1, 8); + setDelay(QueueStatistics, -1, 8); + } + + void bufferReady(Buffer *buffer); + + void setDelay(unsigned int type, int frame, int msdelay); +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_RKISP1_H__ */ diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp new file mode 100644 index 0000000000000000..ca0cd96711d84244 --- /dev/null +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline handler for Rockchip ISP1 + */ + +#include "rkisp1.h" + +#include "log.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(RkISP1) + +void RkISP1ActionSetSensor::run() +{ + sensor_->setControls(&controls_); +} + +void RkISP1ActionQueueBuffer::run() +{ + int ret = device_->queueBuffer(buffer_); + if (ret < 0) + LOG(RkISP1, Error) << "Failed to queue buffer"; +} + +void RkISP1Timeline::bufferReady(Buffer *buffer) +{ + /* + * Calculate SOE by taking the end of DMA set by the kernel and applying + * the time offsets provieded by the IPA to find the best estimate of + * SOE. + * + * NOTE: Make sure the IPA do not set a frame offset for the SOE action + * type as the frame interval is calculated using the interval between + * two SOE events. So using a frame interval in the SOE estimate creates + * a recursion. + */ + + ASSERT(frameOffset(SOE) == 0); + + utils::time_point soe = std::chrono::time_point() + + std::chrono::nanoseconds(buffer->timestamp()) + + timeOffset(SOE); + + notifyStartOfExposure(buffer->sequence(), soe); +} + +void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay) +{ + utils::duration delay = std::chrono::milliseconds(msdelay); + setRawDelay(type, frame, delay); +} + +} /* namespace libcamera */