@@ -8,9 +8,11 @@
#include <algorithm>
#include <array>
#include <iomanip>
+#include <map>
#include <memory>
#include <numeric>
#include <queue>
+#include <vector>
#include <linux/media-bus-format.h>
#include <linux/rkisp1-config.h>
@@ -42,6 +44,7 @@
#include "libcamera/internal/v4l2_subdevice.h"
#include "libcamera/internal/v4l2_videodevice.h"
+#include "rkisp1_dewarper.h"
#include "rkisp1_path.h"
namespace libcamera {
@@ -62,6 +65,8 @@ struct RkISP1FrameInfo {
bool paramDequeued;
bool metadataProcessed;
+
+ std::optional<Rectangle> scalerCrop;
};
class RkISP1Frames
@@ -165,6 +170,8 @@ public:
private:
static constexpr Size kRkISP1PreviewSize = { 1920, 1080 };
+ static constexpr Size kMinDewarpSize = { 176, 144 };
+ static constexpr Size kMaxDewarpSize = { 4096, 3072 };
RkISP1CameraData *cameraData(Camera *camera)
{
@@ -181,6 +188,7 @@ private:
void bufferReady(FrameBuffer *buffer);
void paramReady(FrameBuffer *buffer);
void statReady(FrameBuffer *buffer);
+ void dewarpBufferReady(FrameBuffer *buffer);
void frameStart(uint32_t sequence);
int allocateBuffers(Camera *camera);
@@ -200,6 +208,13 @@ private:
RkISP1MainPath mainPath_;
RkISP1SelfPath selfPath_;
+ std::unique_ptr<RkISP1Dewarper> dewarper_;
+ std::map<unsigned int, FrameBuffer *> dewarpOutputBufs_;
+
+ /* Internal buffers used when dewarper is being used */
+ std::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_;
+ std::queue<FrameBuffer *> availableMainPathBuffers_;
+
std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_;
std::vector<std::unique_ptr<FrameBuffer>> statBuffers_;
std::queue<FrameBuffer *> availableParamBuffers_;
@@ -222,6 +237,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req
FrameBuffer *paramBuffer = nullptr;
FrameBuffer *statBuffer = nullptr;
+ FrameBuffer *mainPathBuffer = nullptr;
+ FrameBuffer *selfPathBuffer = nullptr;
if (!isRaw) {
if (pipe_->availableParamBuffers_.empty()) {
@@ -239,10 +256,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req
statBuffer = pipe_->availableStatBuffers_.front();
pipe_->availableStatBuffers_.pop();
+
+ if (pipe_->dewarper_) {
+ mainPathBuffer = pipe_->availableMainPathBuffers_.front();
+ pipe_->availableMainPathBuffers_.pop();
+ }
}
- FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_);
- FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_);
+ if (!mainPathBuffer)
+ mainPathBuffer = request->findBuffer(&data->mainPathStream_);
+ selfPathBuffer = request->findBuffer(&data->selfPathStream_);
RkISP1FrameInfo *info = new RkISP1FrameInfo;
@@ -268,6 +291,7 @@ int RkISP1Frames::destroy(unsigned int frame)
pipe_->availableParamBuffers_.push(info->paramBuffer);
pipe_->availableStatBuffers_.push(info->statBuffer);
+ pipe_->availableMainPathBuffers_.push(info->mainPathBuffer);
frameInfo_.erase(info->frame);
@@ -283,6 +307,7 @@ void RkISP1Frames::clear()
pipe_->availableParamBuffers_.push(info->paramBuffer);
pipe_->availableStatBuffers_.push(info->statBuffer);
+ pipe_->availableMainPathBuffers_.push(info->mainPathBuffer);
delete info;
}
@@ -785,12 +810,23 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
<< " crop " << rect;
std::map<unsigned int, IPAStream> streamConfig;
+ std::vector<std::reference_wrapper<StreamConfiguration>> outputCfgs;
for (const StreamConfiguration &cfg : *config) {
if (cfg.stream() == &data->mainPathStream_) {
ret = mainPath_.configure(cfg, format);
streamConfig[0] = IPAStream(cfg.pixelFormat,
cfg.size);
+ /* Configure dewarp */
+ if (dewarper_) {
+ /*
+ * \todo Converter::configure() API should be changed
+ * to use std::vector<std::reference_wrapper<const StreamConfiguration>> ?
+ */
+ outputCfgs.push_back(const_cast<StreamConfiguration &>(cfg));
+ dewarper_->configure(cfg, outputCfgs);
+ }
+
} else if (hasSelfPath_) {
ret = selfPath_.configure(cfg, format);
streamConfig[1] = IPAStream(cfg.pixelFormat,
@@ -839,6 +875,9 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S
RkISP1CameraData *data = cameraData(camera);
unsigned int count = stream->configuration().bufferCount;
+ if (!isRaw_ && dewarper_)
+ return dewarper_->exportBuffers(0, count, buffers);
+
if (stream == &data->mainPathStream_)
return mainPath_.exportBuffers(count, buffers);
else if (hasSelfPath_ && stream == &data->selfPathStream_)
@@ -866,6 +905,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)
ret = stat_->allocateBuffers(maxCount, &statBuffers_);
if (ret < 0)
goto error;
+
+ /* If the dewarper is being used, allocate internal buffers for ISP */
+ if (dewarper_) {
+ ret = mainPath_.allocateBuffers(maxCount, &mainPathBuffers_);
+ if (ret < 0)
+ goto error;
+
+ for (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_)
+ availableMainPathBuffers_.push(buffer.get());
+ }
}
for (std::unique_ptr<FrameBuffer> &buffer : paramBuffers_) {
@@ -889,6 +938,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)
error:
paramBuffers_.clear();
statBuffers_.clear();
+ mainPathBuffers_.clear();
return ret;
}
@@ -903,8 +953,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera)
while (!availableParamBuffers_.empty())
availableParamBuffers_.pop();
+ while (!availableMainPathBuffers_.empty())
+ availableMainPathBuffers_.pop();
+
paramBuffers_.clear();
statBuffers_.clear();
+ mainPathBuffers_.clear();
std::vector<unsigned int> ids;
for (IPABuffer &ipabuf : data->ipaBuffers_)
@@ -919,6 +973,9 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera)
if (stat_->releaseBuffers())
LOG(RkISP1, Error) << "Failed to release stat buffers";
+ if (mainPath_.releaseBuffers())
+ LOG(RkISP1, Error) << "Failed to release main path buffers";
+
return 0;
}
@@ -961,6 +1018,14 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL
<< "Failed to start statistics " << camera->id();
return ret;
}
+
+ if (dewarper_) {
+ ret = dewarper_->start();
+ if (ret) {
+ LOG(RkISP1, Error) << "Failed to start dewarper";
+ return ret;
+ }
+ }
}
if (data->mainPath_->isEnabled()) {
@@ -1015,6 +1080,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera)
if (ret)
LOG(RkISP1, Warning)
<< "Failed to stop parameters for " << camera->id();
+
+ if (dewarper_)
+ dewarper_->stop();
}
ASSERT(data->queuedRequests_.empty());
@@ -1045,6 +1113,13 @@ int PipelineHandlerRkISP1::queueRequestDevice(Camera *camera, Request *request)
info->paramBuffer->cookie());
}
+ const auto &crop = request->controls().get(controls::ScalerCrop);
+ if (crop && !isRaw_) {
+ int ret = dewarper_->setScalerCrop(0, crop.value());
+ if (!ret)
+ info->scalerCrop = crop;
+ }
+
data->frame_++;
return 0;
@@ -1110,6 +1185,13 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)
{
ControlInfoMap::Map rkisp1Controls;
+ if (dewarper_) {
+ Rectangle maxCrop(kMaxDewarpSize);
+ Rectangle minCrop = kMinDewarpSize.centeredTo(maxCrop.center());
+
+ rkisp1Controls[&controls::ScalerCrop] = ControlInfo(minCrop, maxCrop, maxCrop);
+ }
+
/* Add the IPA registered controls to list of camera controls. */
for (const auto &ipaControl : data->ipaControls_)
rkisp1Controls[ipaControl.first] = ipaControl.second;
@@ -1173,6 +1255,7 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
{
+ std::shared_ptr<MediaDevice> dwpMediaDevice;
const MediaPad *pad;
DeviceMatch dm("rkisp1");
@@ -1237,6 +1320,26 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
+ /* If dewarper is present, create its instance. */
+ DeviceMatch dwp("dw100");
+ dwp.add("dw100-source");
+ dwp.add("dw100-sink");
+
+ dwpMediaDevice = enumerator->search(dwp);
+ if (dwpMediaDevice) {
+ dewarper_ = std::make_unique<RkISP1Dewarper>(std::move(dwpMediaDevice));
+ if (dewarper_->isValid()) {
+ dewarper_->outputBufferReady.connect(
+ this, &PipelineHandlerRkISP1::dewarpBufferReady);
+
+ LOG(RkISP1, Info) << "Using DW100 dewarper " << dewarper_->deviceNode();
+ } else {
+ LOG(RkISP1, Debug) << "Found DW100 dewarper " << dewarper_->deviceNode()
+ << " but invalid";
+ dewarper_.reset();
+ }
+ }
+
/*
* Enumerate all sensors connected to the ISP and create one
* camera instance for each of them.
@@ -1283,7 +1386,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)
return;
const FrameMetadata &metadata = buffer->metadata();
- Request *request = buffer->request();
+ Request *request = info->request;
if (metadata.status != FrameMetadata::FrameCancelled) {
/*
@@ -1300,11 +1403,36 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)
data->delayedCtrls_->get(metadata.sequence);
data->ipa_->processStatsBuffer(info->frame, 0, ctrls);
}
+
} else {
if (isRaw_)
info->metadataProcessed = true;
}
+ if (dewarper_) {
+ dewarpOutputBufs_[0] = request->findBuffer(&data->mainPathStream_);
+
+ dewarper_->queueBuffers(buffer, dewarpOutputBufs_);
+ return;
+ }
+
+ completeBuffer(request, buffer);
+ tryCompleteRequest(info);
+}
+
+void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer)
+{
+ ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
+ Request *request = buffer->request();
+
+ RkISP1FrameInfo *info = data->frameInfo_.find(buffer->request());
+ if (!info)
+ return;
+
+ if (info->scalerCrop)
+ request->metadata().set(controls::ScalerCrop, info->scalerCrop.value());
+
completeBuffer(request, buffer);
tryCompleteRequest(info);
}
@@ -56,7 +56,7 @@ const std::map<PixelFormat, uint32_t> formatToMediaBus = {
RkISP1Path::RkISP1Path(const char *name, const Span<const PixelFormat> &formats,
const Size &minResolution, const Size &maxResolution)
- : name_(name), running_(false), formats_(formats),
+ : name_(name), running_(false), internalBufs_(false), formats_(formats),
minResolution_(minResolution), maxResolution_(maxResolution),
link_(nullptr)
{
@@ -402,10 +402,12 @@ int RkISP1Path::start()
if (running_)
return -EBUSY;
- /* \todo Make buffer count user configurable. */
- ret = video_->importBuffers(RKISP1_BUFFER_COUNT);
- if (ret)
- return ret;
+ if (!internalBufs_) {
+ /* \todo Make buffer count user configurable. */
+ ret = video_->importBuffers(RKISP1_BUFFER_COUNT);
+ if (ret)
+ return ret;
+ }
ret = video_->streamOn();
if (ret) {
@@ -55,6 +55,19 @@ public:
return video_->exportBuffers(bufferCount, buffers);
}
+ int allocateBuffers(unsigned int bufferCount,
+ std::vector<std::unique_ptr<FrameBuffer>> *buffers)
+ {
+ internalBufs_ = true;
+ return video_->allocateBuffers(bufferCount, buffers);
+ }
+
+ int releaseBuffers()
+ {
+ ASSERT(internalBufs_);
+ return video_->releaseBuffers();
+ }
+
int start();
void stop();
@@ -68,6 +81,7 @@ private:
const char *name_;
bool running_;
+ bool internalBufs_;
const Span<const PixelFormat> formats_;
std::set<PixelFormat> streamFormats_;
Plumb the DW100 RkISP1Dewarper in the rkisp1 pipeline handler. If the dewarper is found, it is instantiated and buffers are exported from it, instead of RkISP1Path. Internal buffers are allocated for the RkISP1Path in case where dewarper is going to be used. The RKISP1 pipeline handler now supports scaler crop control through dewarper. Register the ScalerCrop control for the cameras created in the RKISP1 pipeline handler. Signed-off-by: Umang Jain <umang.jain@ideasonboard.com> --- src/libcamera/pipeline/rkisp1/rkisp1.cpp | 134 +++++++++++++++++- src/libcamera/pipeline/rkisp1/rkisp1_path.cpp | 12 +- src/libcamera/pipeline/rkisp1/rkisp1_path.h | 14 ++ 3 files changed, 152 insertions(+), 8 deletions(-)