| Message ID | 20240926093623.94136-5-umang.jain@ideasonboard.com |
|---|---|
| State | Accepted |
| Headers | show |
| Series |
|
| Related | show |
Hi Umang, Thank you for the patch. On Thu, Sep 26, 2024 at 03:06:23PM +0530, Umang Jain wrote: > Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline > handler. If the dewarper is found, it is instantiated and buffers are > exported from it, instead of RkISP1Path. Internal buffers are allocated > for the RkISP1Path in case where dewarper is going to be used. > > The RKISP1 pipeline handler now supports scaler crop control through > the converter. Register the ScalerCrop control for the cameras created > in the RKISP1 pipeline handler. > > Signed-off-by: Umang Jain <umang.jain@ideasonboard.com> The only thing where I'm uncertain is the implementation of the ScalerCrop. As we know, the scaler crop implementation of the dw100 kernel driver is a bit limited and results in unexpected crops beeing applied. But I think that can/should be fixed in later patches where the vertex map can be used for that. So I think we should merge that in. Reviewed-by: Stefan Klug <stefan.klug@ideasonboard.com> Cheers, Stefan > --- > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 186 ++++++++++++++++++++++- > 1 file changed, 179 insertions(+), 7 deletions(-) > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp > index 651258e3..84c2796e 100644 > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp > @@ -6,10 +6,12 @@ > */ > > #include <algorithm> > +#include <map> > #include <memory> > #include <numeric> > #include <optional> > #include <queue> > +#include <vector> > > #include <linux/media-bus-format.h> > #include <linux/rkisp1-config.h> > @@ -32,6 +34,7 @@ > > #include "libcamera/internal/camera.h" > #include "libcamera/internal/camera_sensor.h" > +#include "libcamera/internal/converter/converter_v4l2_m2m.h" > #include "libcamera/internal/delayed_controls.h" > #include "libcamera/internal/device_enumerator.h" > #include "libcamera/internal/framebuffer.h" > @@ -180,6 +183,7 @@ private: > void bufferReady(FrameBuffer *buffer); > void paramReady(FrameBuffer *buffer); > void statReady(FrameBuffer *buffer); > + void dewarpBufferReady(FrameBuffer *buffer); > void frameStart(uint32_t sequence); > > int allocateBuffers(Camera *camera); > @@ -199,6 +203,15 @@ private: > RkISP1MainPath mainPath_; > RkISP1SelfPath selfPath_; > > + std::unique_ptr<V4L2M2MConverter> dewarper_; > + bool useDewarper_; > + > + std::optional<Rectangle> activeCrop_; > + > + /* Internal buffers used when dewarper is being used */ > + std::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_; > + std::queue<FrameBuffer *> availableMainPathBuffers_; > + > std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_; > std::vector<std::unique_ptr<FrameBuffer>> statBuffers_; > std::queue<FrameBuffer *> availableParamBuffers_; > @@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req > > FrameBuffer *paramBuffer = nullptr; > FrameBuffer *statBuffer = nullptr; > + FrameBuffer *mainPathBuffer = nullptr; > + FrameBuffer *selfPathBuffer = nullptr; > > if (!isRaw) { > if (pipe_->availableParamBuffers_.empty()) { > @@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req > > statBuffer = pipe_->availableStatBuffers_.front(); > pipe_->availableStatBuffers_.pop(); > + > + if (pipe_->useDewarper_) { > + mainPathBuffer = pipe_->availableMainPathBuffers_.front(); > + pipe_->availableMainPathBuffers_.pop(); > + } > } > > - FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_); > - FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_); > + if (!mainPathBuffer) > + mainPathBuffer = request->findBuffer(&data->mainPathStream_); > + selfPathBuffer = request->findBuffer(&data->selfPathStream_); > > RkISP1FrameInfo *info = new RkISP1FrameInfo; > > @@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame) > > pipe_->availableParamBuffers_.push(info->paramBuffer); > pipe_->availableStatBuffers_.push(info->statBuffer); > + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); > > frameInfo_.erase(info->frame); > > @@ -282,6 +304,7 @@ void RkISP1Frames::clear() > > pipe_->availableParamBuffers_.push(info->paramBuffer); > pipe_->availableStatBuffers_.push(info->statBuffer); > + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); > > delete info; > } > @@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() > */ > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) > - : PipelineHandler(manager), hasSelfPath_(true) > + : PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false) > { > } > > @@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) > << " crop " << rect; > > std::map<unsigned int, IPAStream> streamConfig; > + std::vector<std::reference_wrapper<StreamConfiguration>> outputCfgs; > > for (const StreamConfiguration &cfg : *config) { > if (cfg.stream() == &data->mainPathStream_) { > ret = mainPath_.configure(cfg, format); > streamConfig[0] = IPAStream(cfg.pixelFormat, > cfg.size); > + /* Configure dewarp */ > + if (dewarper_ && !isRaw_) { > + outputCfgs.push_back(const_cast<StreamConfiguration &>(cfg)); > + ret = dewarper_->configure(cfg, outputCfgs); > + useDewarper_ = ret ? false : true; > + } > } else if (hasSelfPath_) { > ret = selfPath_.configure(cfg, format); > streamConfig[1] = IPAStream(cfg.pixelFormat, > @@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S > RkISP1CameraData *data = cameraData(camera); > unsigned int count = stream->configuration().bufferCount; > > - if (stream == &data->mainPathStream_) > - return mainPath_.exportBuffers(count, buffers); > - else if (hasSelfPath_ && stream == &data->selfPathStream_) > + if (stream == &data->mainPathStream_) { > + /* > + * Currently, i.MX8MP is the only platform with DW100 dewarper. > + * It has mainpath and no self path. Hence, export buffers from > + * dewarper just for the main path stream, for now. > + */ > + if (useDewarper_) > + return dewarper_->exportBuffers(&data->mainPathStream_, count, buffers); > + else > + return mainPath_.exportBuffers(count, buffers); > + } else if (hasSelfPath_ && stream == &data->selfPathStream_) { > return selfPath_.exportBuffers(count, buffers); > + } > > return -EINVAL; > } > @@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) > ret = stat_->allocateBuffers(maxCount, &statBuffers_); > if (ret < 0) > goto error; > + > + /* If the dewarper is being used, allocate internal buffers for ISP. */ > + if (useDewarper_) { > + ret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_); > + if (ret < 0) > + goto error; > + > + for (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_) > + availableMainPathBuffers_.push(buffer.get()); > + } > } > > for (std::unique_ptr<FrameBuffer> &buffer : paramBuffers_) { > @@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) > error: > paramBuffers_.clear(); > statBuffers_.clear(); > + mainPathBuffers_.clear(); > > return ret; > } > @@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera) > while (!availableParamBuffers_.empty()) > availableParamBuffers_.pop(); > > + while (!availableMainPathBuffers_.empty()) > + availableMainPathBuffers_.pop(); > + > paramBuffers_.clear(); > statBuffers_.clear(); > + mainPathBuffers_.clear(); > > std::vector<unsigned int> ids; > for (IPABuffer &ipabuf : data->ipaBuffers_) > @@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL > return ret; > } > actions += [&]() { stat_->streamOff(); }; > + > + if (useDewarper_) { > + ret = dewarper_->start(); > + if (ret) { > + LOG(RkISP1, Error) << "Failed to start dewarper"; > + return ret; > + } > + } > + actions += [&]() { dewarper_->stop(); }; > } > > if (data->mainPath_->isEnabled()) { > @@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera) > if (ret) > LOG(RkISP1, Warning) > << "Failed to stop parameters for " << camera->id(); > + > + if (useDewarper_) > + dewarper_->stop(); > } > > ASSERT(data->queuedRequests_.empty()); > @@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data) > { > ControlInfoMap::Map controls; > > + if (dewarper_) { > + std::pair<Rectangle, Rectangle> cropLimits = > + dewarper_->inputCropBounds(&data->mainPathStream_); > + > + controls[&controls::ScalerCrop] = ControlInfo(cropLimits.first, > + cropLimits.second, > + cropLimits.second); > + activeCrop_ = cropLimits.second; > + } > + > /* Add the IPA registered controls to list of camera controls. */ > for (const auto &ipaControl : data->ipaControls_) > controls[ipaControl.first] = ipaControl.second; > @@ -1236,6 +1312,29 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) > stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); > param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); > > + /* If dewarper is present, create its instance. */ > + DeviceMatch dwp("dw100"); > + dwp.add("dw100-source"); > + dwp.add("dw100-sink"); > + > + std::shared_ptr<MediaDevice> dwpMediaDevice = enumerator->search(dwp); > + if (dwpMediaDevice) { > + dewarper_ = std::make_unique<V4L2M2MConverter>(dwpMediaDevice.get()); > + if (dewarper_->isValid()) { > + dewarper_->outputBufferReady.connect( > + this, &PipelineHandlerRkISP1::dewarpBufferReady); > + > + LOG(RkISP1, Info) > + << "Using DW100 dewarper " << dewarper_->deviceNode(); > + } else { > + LOG(RkISP1, Warning) > + << "Found DW100 dewarper " << dewarper_->deviceNode() > + << " but invalid"; > + > + dewarper_.reset(); > + } > + } > + > /* > * Enumerate all sensors connected to the ISP and create one > * camera instance for each of them. > @@ -1282,7 +1381,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) > return; > > const FrameMetadata &metadata = buffer->metadata(); > - Request *request = buffer->request(); > + Request *request = info->request; > > if (metadata.status != FrameMetadata::FrameCancelled) { > /* > @@ -1304,6 +1403,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) > info->metadataProcessed = true; > } > > + if (!useDewarper_) { > + completeBuffer(request, buffer); > + tryCompleteRequest(info); > + > + return; > + } > + > + /* Do not queue cancelled frames to dewarper. */ > + if (metadata.status == FrameMetadata::FrameCancelled) { > + /* > + * i.MX8MP is the only known platform with dewarper. It has > + * no self path. Hence, only main path buffer completion is > + * required. > + * > + * Also, we cannot completeBuffer(request, buffer) as buffer > + * here, is an internal buffer (between ISP and dewarper) and > + * is not associated to the any specific request. The request > + * buffer associated with main path stream is the one that > + * is required to be completed (not the internal buffer). > + */ > + for (auto it : request->buffers()) { > + if (it.first == &data->mainPathStream_) > + completeBuffer(request, it.second); > + } > + > + tryCompleteRequest(info); > + return; > + } > + > + /* Handle scaler crop control. */ > + const auto &crop = request->controls().get(controls::ScalerCrop); > + if (crop) { > + Rectangle appliedRect = crop.value(); > + > + int ret = dewarper_->setInputCrop(&data->mainPathStream_, > + &appliedRect); > + if (!ret && appliedRect != crop.value()) { > + /* > + * If the rectangle is changed by setInputCrop on the > + * dewarper, log a debug message and cache the actual > + * applied rectangle for metadata reporting. > + */ > + LOG(RkISP1, Debug) > + << "Applied rectangle " << appliedRect.toString() > + << " differs from requested " << crop.value().toString(); > + } > + > + activeCrop_ = appliedRect; > + } > + > + /* > + * Queue input and output buffers to the dewarper. The output > + * buffers for the dewarper are the buffers of the request, supplied > + * by the application. > + */ > + int ret = dewarper_->queueBuffers(buffer, request->buffers()); > + if (ret < 0) > + LOG(RkISP1, Error) << "Cannot queue buffers to dewarper: " > + << strerror(-ret); > + > + request->metadata().set(controls::ScalerCrop, activeCrop_.value()); > +} > + > +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer) > +{ > + ASSERT(activeCamera_); > + RkISP1CameraData *data = cameraData(activeCamera_); > + Request *request = buffer->request(); > + > + RkISP1FrameInfo *info = data->frameInfo_.find(buffer->request()); > + if (!info) > + return; > + > completeBuffer(request, buffer); > tryCompleteRequest(info); > } > -- > 2.45.0 >
On Thu, Sep 26, 2024 at 03:06:23PM +0530, Umang Jain wrote: > Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline > handler. If the dewarper is found, it is instantiated and buffers are > exported from it, instead of RkISP1Path. Internal buffers are allocated > for the RkISP1Path in case where dewarper is going to be used. > > The RKISP1 pipeline handler now supports scaler crop control through > the converter. Register the ScalerCrop control for the cameras created > in the RKISP1 pipeline handler. > > Signed-off-by: Umang Jain <umang.jain@ideasonboard.com> Looks good to me. Reviewed-by: Paul Elder <paul.elder@ideasonboard.com> > --- > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 186 ++++++++++++++++++++++- > 1 file changed, 179 insertions(+), 7 deletions(-) > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp > index 651258e3..84c2796e 100644 > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp > @@ -6,10 +6,12 @@ > */ > > #include <algorithm> > +#include <map> > #include <memory> > #include <numeric> > #include <optional> > #include <queue> > +#include <vector> > > #include <linux/media-bus-format.h> > #include <linux/rkisp1-config.h> > @@ -32,6 +34,7 @@ > > #include "libcamera/internal/camera.h" > #include "libcamera/internal/camera_sensor.h" > +#include "libcamera/internal/converter/converter_v4l2_m2m.h" > #include "libcamera/internal/delayed_controls.h" > #include "libcamera/internal/device_enumerator.h" > #include "libcamera/internal/framebuffer.h" > @@ -180,6 +183,7 @@ private: > void bufferReady(FrameBuffer *buffer); > void paramReady(FrameBuffer *buffer); > void statReady(FrameBuffer *buffer); > + void dewarpBufferReady(FrameBuffer *buffer); > void frameStart(uint32_t sequence); > > int allocateBuffers(Camera *camera); > @@ -199,6 +203,15 @@ private: > RkISP1MainPath mainPath_; > RkISP1SelfPath selfPath_; > > + std::unique_ptr<V4L2M2MConverter> dewarper_; > + bool useDewarper_; > + > + std::optional<Rectangle> activeCrop_; > + > + /* Internal buffers used when dewarper is being used */ > + std::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_; > + std::queue<FrameBuffer *> availableMainPathBuffers_; > + > std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_; > std::vector<std::unique_ptr<FrameBuffer>> statBuffers_; > std::queue<FrameBuffer *> availableParamBuffers_; > @@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req > > FrameBuffer *paramBuffer = nullptr; > FrameBuffer *statBuffer = nullptr; > + FrameBuffer *mainPathBuffer = nullptr; > + FrameBuffer *selfPathBuffer = nullptr; > > if (!isRaw) { > if (pipe_->availableParamBuffers_.empty()) { > @@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req > > statBuffer = pipe_->availableStatBuffers_.front(); > pipe_->availableStatBuffers_.pop(); > + > + if (pipe_->useDewarper_) { > + mainPathBuffer = pipe_->availableMainPathBuffers_.front(); > + pipe_->availableMainPathBuffers_.pop(); > + } > } > > - FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_); > - FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_); > + if (!mainPathBuffer) > + mainPathBuffer = request->findBuffer(&data->mainPathStream_); > + selfPathBuffer = request->findBuffer(&data->selfPathStream_); > > RkISP1FrameInfo *info = new RkISP1FrameInfo; > > @@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame) > > pipe_->availableParamBuffers_.push(info->paramBuffer); > pipe_->availableStatBuffers_.push(info->statBuffer); > + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); > > frameInfo_.erase(info->frame); > > @@ -282,6 +304,7 @@ void RkISP1Frames::clear() > > pipe_->availableParamBuffers_.push(info->paramBuffer); > pipe_->availableStatBuffers_.push(info->statBuffer); > + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); > > delete info; > } > @@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() > */ > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) > - : PipelineHandler(manager), hasSelfPath_(true) > + : PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false) > { > } > > @@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) > << " crop " << rect; > > std::map<unsigned int, IPAStream> streamConfig; > + std::vector<std::reference_wrapper<StreamConfiguration>> outputCfgs; > > for (const StreamConfiguration &cfg : *config) { > if (cfg.stream() == &data->mainPathStream_) { > ret = mainPath_.configure(cfg, format); > streamConfig[0] = IPAStream(cfg.pixelFormat, > cfg.size); > + /* Configure dewarp */ > + if (dewarper_ && !isRaw_) { > + outputCfgs.push_back(const_cast<StreamConfiguration &>(cfg)); > + ret = dewarper_->configure(cfg, outputCfgs); > + useDewarper_ = ret ? false : true; > + } > } else if (hasSelfPath_) { > ret = selfPath_.configure(cfg, format); > streamConfig[1] = IPAStream(cfg.pixelFormat, > @@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S > RkISP1CameraData *data = cameraData(camera); > unsigned int count = stream->configuration().bufferCount; > > - if (stream == &data->mainPathStream_) > - return mainPath_.exportBuffers(count, buffers); > - else if (hasSelfPath_ && stream == &data->selfPathStream_) > + if (stream == &data->mainPathStream_) { > + /* > + * Currently, i.MX8MP is the only platform with DW100 dewarper. > + * It has mainpath and no self path. Hence, export buffers from > + * dewarper just for the main path stream, for now. > + */ > + if (useDewarper_) > + return dewarper_->exportBuffers(&data->mainPathStream_, count, buffers); > + else > + return mainPath_.exportBuffers(count, buffers); > + } else if (hasSelfPath_ && stream == &data->selfPathStream_) { > return selfPath_.exportBuffers(count, buffers); > + } > > return -EINVAL; > } > @@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) > ret = stat_->allocateBuffers(maxCount, &statBuffers_); > if (ret < 0) > goto error; > + > + /* If the dewarper is being used, allocate internal buffers for ISP. */ > + if (useDewarper_) { > + ret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_); > + if (ret < 0) > + goto error; > + > + for (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_) > + availableMainPathBuffers_.push(buffer.get()); > + } > } > > for (std::unique_ptr<FrameBuffer> &buffer : paramBuffers_) { > @@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) > error: > paramBuffers_.clear(); > statBuffers_.clear(); > + mainPathBuffers_.clear(); > > return ret; > } > @@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera) > while (!availableParamBuffers_.empty()) > availableParamBuffers_.pop(); > > + while (!availableMainPathBuffers_.empty()) > + availableMainPathBuffers_.pop(); > + > paramBuffers_.clear(); > statBuffers_.clear(); > + mainPathBuffers_.clear(); > > std::vector<unsigned int> ids; > for (IPABuffer &ipabuf : data->ipaBuffers_) > @@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL > return ret; > } > actions += [&]() { stat_->streamOff(); }; > + > + if (useDewarper_) { > + ret = dewarper_->start(); > + if (ret) { > + LOG(RkISP1, Error) << "Failed to start dewarper"; > + return ret; > + } > + } > + actions += [&]() { dewarper_->stop(); }; > } > > if (data->mainPath_->isEnabled()) { > @@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera) > if (ret) > LOG(RkISP1, Warning) > << "Failed to stop parameters for " << camera->id(); > + > + if (useDewarper_) > + dewarper_->stop(); > } > > ASSERT(data->queuedRequests_.empty()); > @@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data) > { > ControlInfoMap::Map controls; > > + if (dewarper_) { > + std::pair<Rectangle, Rectangle> cropLimits = > + dewarper_->inputCropBounds(&data->mainPathStream_); > + > + controls[&controls::ScalerCrop] = ControlInfo(cropLimits.first, > + cropLimits.second, > + cropLimits.second); > + activeCrop_ = cropLimits.second; > + } > + > /* Add the IPA registered controls to list of camera controls. */ > for (const auto &ipaControl : data->ipaControls_) > controls[ipaControl.first] = ipaControl.second; > @@ -1236,6 +1312,29 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) > stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); > param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); > > + /* If dewarper is present, create its instance. */ > + DeviceMatch dwp("dw100"); > + dwp.add("dw100-source"); > + dwp.add("dw100-sink"); > + > + std::shared_ptr<MediaDevice> dwpMediaDevice = enumerator->search(dwp); > + if (dwpMediaDevice) { > + dewarper_ = std::make_unique<V4L2M2MConverter>(dwpMediaDevice.get()); > + if (dewarper_->isValid()) { > + dewarper_->outputBufferReady.connect( > + this, &PipelineHandlerRkISP1::dewarpBufferReady); > + > + LOG(RkISP1, Info) > + << "Using DW100 dewarper " << dewarper_->deviceNode(); > + } else { > + LOG(RkISP1, Warning) > + << "Found DW100 dewarper " << dewarper_->deviceNode() > + << " but invalid"; > + > + dewarper_.reset(); > + } > + } > + > /* > * Enumerate all sensors connected to the ISP and create one > * camera instance for each of them. > @@ -1282,7 +1381,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) > return; > > const FrameMetadata &metadata = buffer->metadata(); > - Request *request = buffer->request(); > + Request *request = info->request; > > if (metadata.status != FrameMetadata::FrameCancelled) { > /* > @@ -1304,6 +1403,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) > info->metadataProcessed = true; > } > > + if (!useDewarper_) { > + completeBuffer(request, buffer); > + tryCompleteRequest(info); > + > + return; > + } > + > + /* Do not queue cancelled frames to dewarper. */ > + if (metadata.status == FrameMetadata::FrameCancelled) { > + /* > + * i.MX8MP is the only known platform with dewarper. It has > + * no self path. Hence, only main path buffer completion is > + * required. > + * > + * Also, we cannot completeBuffer(request, buffer) as buffer > + * here, is an internal buffer (between ISP and dewarper) and > + * is not associated to the any specific request. The request > + * buffer associated with main path stream is the one that > + * is required to be completed (not the internal buffer). > + */ > + for (auto it : request->buffers()) { > + if (it.first == &data->mainPathStream_) > + completeBuffer(request, it.second); > + } > + > + tryCompleteRequest(info); > + return; > + } > + > + /* Handle scaler crop control. */ > + const auto &crop = request->controls().get(controls::ScalerCrop); > + if (crop) { > + Rectangle appliedRect = crop.value(); > + > + int ret = dewarper_->setInputCrop(&data->mainPathStream_, > + &appliedRect); > + if (!ret && appliedRect != crop.value()) { > + /* > + * If the rectangle is changed by setInputCrop on the > + * dewarper, log a debug message and cache the actual > + * applied rectangle for metadata reporting. > + */ > + LOG(RkISP1, Debug) > + << "Applied rectangle " << appliedRect.toString() > + << " differs from requested " << crop.value().toString(); > + } > + > + activeCrop_ = appliedRect; > + } > + > + /* > + * Queue input and output buffers to the dewarper. The output > + * buffers for the dewarper are the buffers of the request, supplied > + * by the application. > + */ > + int ret = dewarper_->queueBuffers(buffer, request->buffers()); > + if (ret < 0) > + LOG(RkISP1, Error) << "Cannot queue buffers to dewarper: " > + << strerror(-ret); > + > + request->metadata().set(controls::ScalerCrop, activeCrop_.value()); > +} > + > +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer) > +{ > + ASSERT(activeCamera_); > + RkISP1CameraData *data = cameraData(activeCamera_); > + Request *request = buffer->request(); > + > + RkISP1FrameInfo *info = data->frameInfo_.find(buffer->request()); > + if (!info) > + return; > + > completeBuffer(request, buffer); > tryCompleteRequest(info); > } > -- > 2.45.0 >
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index 651258e3..84c2796e 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -6,10 +6,12 @@ */ #include <algorithm> +#include <map> #include <memory> #include <numeric> #include <optional> #include <queue> +#include <vector> #include <linux/media-bus-format.h> #include <linux/rkisp1-config.h> @@ -32,6 +34,7 @@ #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/converter/converter_v4l2_m2m.h" #include "libcamera/internal/delayed_controls.h" #include "libcamera/internal/device_enumerator.h" #include "libcamera/internal/framebuffer.h" @@ -180,6 +183,7 @@ private: void bufferReady(FrameBuffer *buffer); void paramReady(FrameBuffer *buffer); void statReady(FrameBuffer *buffer); + void dewarpBufferReady(FrameBuffer *buffer); void frameStart(uint32_t sequence); int allocateBuffers(Camera *camera); @@ -199,6 +203,15 @@ private: RkISP1MainPath mainPath_; RkISP1SelfPath selfPath_; + std::unique_ptr<V4L2M2MConverter> dewarper_; + bool useDewarper_; + + std::optional<Rectangle> activeCrop_; + + /* Internal buffers used when dewarper is being used */ + std::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_; + std::queue<FrameBuffer *> availableMainPathBuffers_; + std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_; std::vector<std::unique_ptr<FrameBuffer>> statBuffers_; std::queue<FrameBuffer *> availableParamBuffers_; @@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req FrameBuffer *paramBuffer = nullptr; FrameBuffer *statBuffer = nullptr; + FrameBuffer *mainPathBuffer = nullptr; + FrameBuffer *selfPathBuffer = nullptr; if (!isRaw) { if (pipe_->availableParamBuffers_.empty()) { @@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req statBuffer = pipe_->availableStatBuffers_.front(); pipe_->availableStatBuffers_.pop(); + + if (pipe_->useDewarper_) { + mainPathBuffer = pipe_->availableMainPathBuffers_.front(); + pipe_->availableMainPathBuffers_.pop(); + } } - FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_); - FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_); + if (!mainPathBuffer) + mainPathBuffer = request->findBuffer(&data->mainPathStream_); + selfPathBuffer = request->findBuffer(&data->selfPathStream_); RkISP1FrameInfo *info = new RkISP1FrameInfo; @@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame) pipe_->availableParamBuffers_.push(info->paramBuffer); pipe_->availableStatBuffers_.push(info->statBuffer); + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); frameInfo_.erase(info->frame); @@ -282,6 +304,7 @@ void RkISP1Frames::clear() pipe_->availableParamBuffers_.push(info->paramBuffer); pipe_->availableStatBuffers_.push(info->statBuffer); + pipe_->availableMainPathBuffers_.push(info->mainPathBuffer); delete info; } @@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate() */ PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager) - : PipelineHandler(manager), hasSelfPath_(true) + : PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false) { } @@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c) << " crop " << rect; std::map<unsigned int, IPAStream> streamConfig; + std::vector<std::reference_wrapper<StreamConfiguration>> outputCfgs; for (const StreamConfiguration &cfg : *config) { if (cfg.stream() == &data->mainPathStream_) { ret = mainPath_.configure(cfg, format); streamConfig[0] = IPAStream(cfg.pixelFormat, cfg.size); + /* Configure dewarp */ + if (dewarper_ && !isRaw_) { + outputCfgs.push_back(const_cast<StreamConfiguration &>(cfg)); + ret = dewarper_->configure(cfg, outputCfgs); + useDewarper_ = ret ? false : true; + } } else if (hasSelfPath_) { ret = selfPath_.configure(cfg, format); streamConfig[1] = IPAStream(cfg.pixelFormat, @@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S RkISP1CameraData *data = cameraData(camera); unsigned int count = stream->configuration().bufferCount; - if (stream == &data->mainPathStream_) - return mainPath_.exportBuffers(count, buffers); - else if (hasSelfPath_ && stream == &data->selfPathStream_) + if (stream == &data->mainPathStream_) { + /* + * Currently, i.MX8MP is the only platform with DW100 dewarper. + * It has mainpath and no self path. Hence, export buffers from + * dewarper just for the main path stream, for now. + */ + if (useDewarper_) + return dewarper_->exportBuffers(&data->mainPathStream_, count, buffers); + else + return mainPath_.exportBuffers(count, buffers); + } else if (hasSelfPath_ && stream == &data->selfPathStream_) { return selfPath_.exportBuffers(count, buffers); + } return -EINVAL; } @@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) ret = stat_->allocateBuffers(maxCount, &statBuffers_); if (ret < 0) goto error; + + /* If the dewarper is being used, allocate internal buffers for ISP. */ + if (useDewarper_) { + ret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_); + if (ret < 0) + goto error; + + for (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_) + availableMainPathBuffers_.push(buffer.get()); + } } for (std::unique_ptr<FrameBuffer> &buffer : paramBuffers_) { @@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) error: paramBuffers_.clear(); statBuffers_.clear(); + mainPathBuffers_.clear(); return ret; } @@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera) while (!availableParamBuffers_.empty()) availableParamBuffers_.pop(); + while (!availableMainPathBuffers_.empty()) + availableMainPathBuffers_.pop(); + paramBuffers_.clear(); statBuffers_.clear(); + mainPathBuffers_.clear(); std::vector<unsigned int> ids; for (IPABuffer &ipabuf : data->ipaBuffers_) @@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL return ret; } actions += [&]() { stat_->streamOff(); }; + + if (useDewarper_) { + ret = dewarper_->start(); + if (ret) { + LOG(RkISP1, Error) << "Failed to start dewarper"; + return ret; + } + } + actions += [&]() { dewarper_->stop(); }; } if (data->mainPath_->isEnabled()) { @@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera) if (ret) LOG(RkISP1, Warning) << "Failed to stop parameters for " << camera->id(); + + if (useDewarper_) + dewarper_->stop(); } ASSERT(data->queuedRequests_.empty()); @@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data) { ControlInfoMap::Map controls; + if (dewarper_) { + std::pair<Rectangle, Rectangle> cropLimits = + dewarper_->inputCropBounds(&data->mainPathStream_); + + controls[&controls::ScalerCrop] = ControlInfo(cropLimits.first, + cropLimits.second, + cropLimits.second); + activeCrop_ = cropLimits.second; + } + /* Add the IPA registered controls to list of camera controls. */ for (const auto &ipaControl : data->ipaControls_) controls[ipaControl.first] = ipaControl.second; @@ -1236,6 +1312,29 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); + /* If dewarper is present, create its instance. */ + DeviceMatch dwp("dw100"); + dwp.add("dw100-source"); + dwp.add("dw100-sink"); + + std::shared_ptr<MediaDevice> dwpMediaDevice = enumerator->search(dwp); + if (dwpMediaDevice) { + dewarper_ = std::make_unique<V4L2M2MConverter>(dwpMediaDevice.get()); + if (dewarper_->isValid()) { + dewarper_->outputBufferReady.connect( + this, &PipelineHandlerRkISP1::dewarpBufferReady); + + LOG(RkISP1, Info) + << "Using DW100 dewarper " << dewarper_->deviceNode(); + } else { + LOG(RkISP1, Warning) + << "Found DW100 dewarper " << dewarper_->deviceNode() + << " but invalid"; + + dewarper_.reset(); + } + } + /* * Enumerate all sensors connected to the ISP and create one * camera instance for each of them. @@ -1282,7 +1381,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) return; const FrameMetadata &metadata = buffer->metadata(); - Request *request = buffer->request(); + Request *request = info->request; if (metadata.status != FrameMetadata::FrameCancelled) { /* @@ -1304,6 +1403,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) info->metadataProcessed = true; } + if (!useDewarper_) { + completeBuffer(request, buffer); + tryCompleteRequest(info); + + return; + } + + /* Do not queue cancelled frames to dewarper. */ + if (metadata.status == FrameMetadata::FrameCancelled) { + /* + * i.MX8MP is the only known platform with dewarper. It has + * no self path. Hence, only main path buffer completion is + * required. + * + * Also, we cannot completeBuffer(request, buffer) as buffer + * here, is an internal buffer (between ISP and dewarper) and + * is not associated to the any specific request. The request + * buffer associated with main path stream is the one that + * is required to be completed (not the internal buffer). + */ + for (auto it : request->buffers()) { + if (it.first == &data->mainPathStream_) + completeBuffer(request, it.second); + } + + tryCompleteRequest(info); + return; + } + + /* Handle scaler crop control. */ + const auto &crop = request->controls().get(controls::ScalerCrop); + if (crop) { + Rectangle appliedRect = crop.value(); + + int ret = dewarper_->setInputCrop(&data->mainPathStream_, + &appliedRect); + if (!ret && appliedRect != crop.value()) { + /* + * If the rectangle is changed by setInputCrop on the + * dewarper, log a debug message and cache the actual + * applied rectangle for metadata reporting. + */ + LOG(RkISP1, Debug) + << "Applied rectangle " << appliedRect.toString() + << " differs from requested " << crop.value().toString(); + } + + activeCrop_ = appliedRect; + } + + /* + * Queue input and output buffers to the dewarper. The output + * buffers for the dewarper are the buffers of the request, supplied + * by the application. + */ + int ret = dewarper_->queueBuffers(buffer, request->buffers()); + if (ret < 0) + LOG(RkISP1, Error) << "Cannot queue buffers to dewarper: " + << strerror(-ret); + + request->metadata().set(controls::ScalerCrop, activeCrop_.value()); +} + +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer) +{ + ASSERT(activeCamera_); + RkISP1CameraData *data = cameraData(activeCamera_); + Request *request = buffer->request(); + + RkISP1FrameInfo *info = data->frameInfo_.find(buffer->request()); + if (!info) + return; + completeBuffer(request, buffer); tryCompleteRequest(info); }
Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline handler. If the dewarper is found, it is instantiated and buffers are exported from it, instead of RkISP1Path. Internal buffers are allocated for the RkISP1Path in case where dewarper is going to be used. The RKISP1 pipeline handler now supports scaler crop control through the converter. Register the ScalerCrop control for the cameras created in the RKISP1 pipeline handler. Signed-off-by: Umang Jain <umang.jain@ideasonboard.com> --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 186 ++++++++++++++++++++++- 1 file changed, 179 insertions(+), 7 deletions(-)