[libcamera-devel,v3,13/13] libcamera: pipeline: rkisp1: Attach to an IPA

2037 diff mbox series

diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build
index f1cc4046b5d064cb..d04fb45223e72fa1 100644
--- a/src/libcamera/pipeline/rkisp1/meson.build
+++ b/src/libcamera/pipeline/rkisp1/meson.build
@@ -1,3 +1,4 @@ 
 libcamera_sources += files([
     'rkisp1.cpp',
+    'timeline.cpp',
 ])
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index de4ab523d0e4fe36..5ca2c93bc7e70100 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -5,11 +5,12 @@ 
  * rkisp1.cpp - Pipeline handler for Rockchip ISP1
  */
 
+#include "rkisp1.h"
+
 #include <algorithm>
 #include <array>
 #include <iomanip>
 #include <memory>
-#include <vector>
 
 #include <linux/media-bus-format.h>
 
@@ -17,8 +18,8 @@ 
 #include <libcamera/request.h>
 #include <libcamera/stream.h>
 
-#include "camera_sensor.h"
 #include "device_enumerator.h"
+#include "ipa_manager.h"
 #include "log.h"
 #include "media_device.h"
 #include "pipeline_handler.h"
@@ -34,7 +35,7 @@  class RkISP1CameraData : public CameraData
 {
 public:
 	RkISP1CameraData(PipelineHandler *pipe)
-		: CameraData(pipe), sensor_(nullptr)
+		: CameraData(pipe), sensor_(nullptr), frame_(0)
 	{
 	}
 
@@ -43,8 +44,19 @@  public:
 		delete sensor_;
 	}
 
+	int loadIPA() override;
+
 	Stream stream_;
 	CameraSensor *sensor_;
+	unsigned int frame_;
+	std::map<unsigned int, Request *> frameInfo_;
+	RkISP1Timeline timeline_;
+
+private:
+	void updateSensor(unsigned int frame, V4L2ControlList controls);
+	void queueBuffer(unsigned int frame, unsigned int type,
+			 unsigned int id);
+	void metaDataReady(unsigned int frame, IPAMetaData metaData);
 };
 
 class RkISP1CameraConfiguration : public CameraConfiguration
@@ -99,18 +111,94 @@  private:
 			PipelineHandler::cameraData(camera));
 	}
 
+	friend RkISP1CameraData;
+
 	int initLinks();
 	int createCamera(MediaEntity *sensor);
+	void tryCompleteRequest(Request *request);
 	void bufferReady(Buffer *buffer);
+	void paramReady(Buffer *buffer);
+	void statReady(Buffer *buffer);
 
 	MediaDevice *media_;
 	V4L2Subdevice *dphy_;
 	V4L2Subdevice *isp_;
 	V4L2VideoDevice *video_;
+	V4L2VideoDevice *param_;
+	V4L2VideoDevice *stat_;
+
+	BufferPool paramPool_;
+	BufferPool statPool_;
+
+	std::map<unsigned int, Buffer *> paramBuffers_;
+	std::map<unsigned int, Buffer *> statBuffers_;
 
 	Camera *activeCamera_;
 };
 
+int RkISP1CameraData::loadIPA()
+{
+	ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);
+	if (!ipa_)
+		return -ENOENT;
+
+	ipa_->updateSensor.connect(this,
+				   &RkISP1CameraData::updateSensor);
+	ipa_->queueBuffer.connect(this,
+				  &RkISP1CameraData::queueBuffer);
+	ipa_->setDelay.connect(&timeline_,
+			       &RkISP1Timeline::setDelay);
+	ipa_->metaDataReady.connect(this,
+				    &RkISP1CameraData::metaDataReady);
+
+	return 0;
+}
+
+void RkISP1CameraData::updateSensor(unsigned int frame, V4L2ControlList controls)
+{
+	timeline_.scheduleAction(new RkISP1ActionSetSensor(frame, sensor_, controls));
+}
+
+void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type,
+				   unsigned int id)
+{
+	PipelineHandlerRkISP1 *pipe =
+		static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+	RkISP1ActionType acttype;
+	V4L2VideoDevice *device;
+	Buffer *buffer;
+	switch (type) {
+	case BUFFER_PARAM:
+		acttype = QueueParameters;
+		device = pipe->param_;
+		buffer = pipe->paramBuffers_[id];
+		break;
+	case BUFFER_STAT:
+		acttype = QueueStatistics;
+		device = pipe->stat_;
+		buffer = pipe->statBuffers_[id];
+		break;
+	default:
+		LOG(RkISP1, Error) << "Unkown IPA buffer type " << type;
+		return;
+	}
+
+	timeline_.scheduleAction(new RkISP1ActionQueueBuffer(frame, acttype,
+							     device, buffer));
+}
+
+void RkISP1CameraData::metaDataReady(unsigned int frame, IPAMetaData metaData)
+{
+	Request *request = frameInfo_[frame];
+	PipelineHandlerRkISP1 *pipe =
+		static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+	pipe->processMetaData(request, metaData);
+
+	pipe->tryCompleteRequest(request);
+}
+
 RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,
 						     RkISP1CameraData *data)
 	: CameraConfiguration()
@@ -202,12 +290,14 @@  CameraConfiguration::Status RkISP1CameraConfiguration::validate()
 
 PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)
 	: PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),
-	  video_(nullptr)
+	  video_(nullptr), param_(nullptr), stat_(nullptr)
 {
 }
 
 PipelineHandlerRkISP1::~PipelineHandlerRkISP1()
 {
+	delete param_;
+	delete stat_;
 	delete video_;
 	delete isp_;
 	delete dphy_;
@@ -317,6 +407,20 @@  int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
 	if (ret)
 		return ret;
 
+	V4L2DeviceFormat paramFormat = {};
+	paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;
+
+	ret = param_->setFormat(&paramFormat);
+	if (ret)
+		return ret;
+
+	V4L2DeviceFormat statFormat = {};
+	statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;
+
+	ret = stat_->setFormat(&statFormat);
+	if (ret)
+		return ret;
+
 	if (outputFormat.size != cfg.size ||
 	    outputFormat.fourcc != cfg.pixelFormat) {
 		LOG(RkISP1, Error)
@@ -332,31 +436,99 @@  int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
 int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,
 					   const std::set<Stream *> &streams)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	Stream *stream = *streams.begin();
+	int ret;
 
 	if (stream->memoryType() == InternalMemory)
-		return video_->exportBuffers(&stream->bufferPool());
+		ret = video_->exportBuffers(&stream->bufferPool());
 	else
-		return video_->importBuffers(&stream->bufferPool());
+		ret = video_->importBuffers(&stream->bufferPool());
+
+	if (ret)
+		return ret;
+
+	paramPool_.createBuffers(stream->configuration().bufferCount + 1);
+	ret = param_->exportBuffers(&paramPool_);
+	if (ret) {
+		video_->releaseBuffers();
+		return ret;
+	}
+
+	statPool_.createBuffers(stream->configuration().bufferCount + 1);
+	ret = stat_->exportBuffers(&statPool_);
+	if (ret) {
+		param_->releaseBuffers();
+		video_->releaseBuffers();
+		return ret;
+	}
+
+	for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+		paramBuffers_[i] = new Buffer(i);
+		statBuffers_[i] = new Buffer(i);
+	}
+
+	data->ipa_->initBuffers(BUFFER_PARAM, paramPool_.buffers());
+	data->ipa_->initBuffers(BUFFER_STAT, statPool_.buffers());
+
+	return ret;
 }
 
 int PipelineHandlerRkISP1::freeBuffers(Camera *camera,
 				       const std::set<Stream *> &streams)
 {
+	for (auto it : paramBuffers_)
+		delete it.second;
+
+	paramBuffers_.clear();
+
+	for (auto it : statBuffers_)
+		delete it.second;
+
+	statBuffers_.clear();
+
+	if (param_->releaseBuffers())
+		LOG(RkISP1, Error) << "Failed to release parameters buffers";
+
+	if (stat_->releaseBuffers())
+		LOG(RkISP1, Error) << "Failed to release stat buffers";
+
 	if (video_->releaseBuffers())
-		LOG(RkISP1, Error) << "Failed to release buffers";
+		LOG(RkISP1, Error) << "Failed to release video buffers";
 
 	return 0;
 }
 
 int PipelineHandlerRkISP1::start(Camera *camera)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	int ret;
 
+	data->ipa_->initSensor(data->sensor_->controls());
+
+	ret = param_->streamOn();
+	if (ret) {
+		LOG(RkISP1, Error)
+			<< "Failed to start parameters " << camera->name();
+		return ret;
+	}
+
+	ret = stat_->streamOn();
+	if (ret) {
+		param_->streamOff();
+		LOG(RkISP1, Error)
+			<< "Failed to start statistics " << camera->name();
+		return ret;
+	}
+
 	ret = video_->streamOn();
-	if (ret)
+	if (ret) {
+		param_->streamOff();
+		stat_->streamOff();
+
 		LOG(RkISP1, Error)
 			<< "Failed to start camera " << camera->name();
+	}
 
 	activeCamera_ = camera;
 
@@ -365,6 +537,7 @@  int PipelineHandlerRkISP1::start(Camera *camera)
 
 void PipelineHandlerRkISP1::stop(Camera *camera)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	int ret;
 
 	ret = video_->streamOff();
@@ -372,6 +545,18 @@  void PipelineHandlerRkISP1::stop(Camera *camera)
 		LOG(RkISP1, Warning)
 			<< "Failed to stop camera " << camera->name();
 
+	ret = stat_->streamOff();
+	if (ret)
+		LOG(RkISP1, Warning)
+			<< "Failed to stop statistics " << camera->name();
+
+	ret = param_->streamOff();
+	if (ret)
+		LOG(RkISP1, Warning)
+			<< "Failed to stop parameters " << camera->name();
+
+	data->timeline_.reset();
+
 	activeCamera_ = nullptr;
 }
 
@@ -387,12 +572,17 @@  int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)
 		return -ENOENT;
 	}
 
-	int ret = video_->queueBuffer(buffer);
-	if (ret < 0)
-		return ret;
-
 	PipelineHandler::queueRequest(camera, request);
 
+	data->frameInfo_[data->frame_] = request;
+	data->ipa_->queueRequest(data->frame_, request->controls());
+
+	data->timeline_.scheduleAction(new RkISP1ActionQueueBuffer(data->frame_,
+								   QueueVideo,
+								   video_,
+								   buffer));
+	data->frame_++;
+
 	return 0;
 }
 
@@ -435,6 +625,10 @@  int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
 	std::unique_ptr<RkISP1CameraData> data =
 		utils::make_unique<RkISP1CameraData>(this);
 
+	data->controlInfo_.emplace(std::piecewise_construct,
+				   std::forward_as_tuple(AeEnable),
+				   std::forward_as_tuple(AeEnable, false, true));
+
 	data->sensor_ = new CameraSensor(sensor);
 	ret = data->sensor_->init();
 	if (ret)
@@ -478,7 +672,17 @@  bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
 	if (video_->open() < 0)
 		return false;
 
+	stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics");
+	if (stat_->open() < 0)
+		return false;
+
+	param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params");
+	if (param_->open() < 0)
+		return false;
+
 	video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);
+	stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
+	param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
 
 	/* Configure default links. */
 	if (initLinks() < 0) {
@@ -504,13 +708,46 @@  bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
  * Buffer Handling
  */
 
+void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)
+{
+	if (request->hasPendingBuffers())
+		return;
+
+	if (!request->metaData().ready)
+		return;
+
+	completeRequest(activeCamera_, request);
+}
+
 void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
 {
 	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
 	Request *request = buffer->request();
 
+	data->timeline_.bufferReady(buffer);
+
+	if (data->frame_ <= buffer->sequence())
+		data->frame_ = buffer->sequence() + 1;
+
 	completeBuffer(activeCamera_, request, buffer);
-	completeRequest(activeCamera_, request);
+	tryCompleteRequest(request);
+}
+
+void PipelineHandlerRkISP1::paramReady(Buffer *buffer)
+{
+	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
+
+	data->ipa_->signalBuffer(BUFFER_PARAM, buffer->index());
+}
+
+void PipelineHandlerRkISP1::statReady(Buffer *buffer)
+{
+	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
+
+	data->ipa_->signalBuffer(BUFFER_STAT, buffer->index());
 }
 
 REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h
new file mode 100644
index 0000000000000000..aea0eaa6bc838755
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.h
@@ -0,0 +1,81 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * rkisp1.h - Pipeline handler for Rockchip ISP1
+ */
+#ifndef __LIBCAMERA_RKISP1_H__
+#define __LIBCAMERA_RKISP1_H__
+
+#include "timeline.h"
+
+#include <libcamera/buffer.h>
+
+#include "camera_sensor.h"
+#include "v4l2_videodevice.h"
+
+#define BUFFER_PARAM 1
+#define BUFFER_STAT 2
+
+namespace libcamera {
+
+enum RkISP1ActionType {
+	SetSensor,
+	SOE,
+	QueueVideo,
+	QueueParameters,
+	QueueStatistics,
+};
+
+class RkISP1ActionSetSensor : public FrameAction
+{
+public:
+	RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)
+		: FrameAction(SetSensor, frame), sensor_(sensor), controls_(controls) {}
+
+protected:
+	void run() override;
+
+private:
+	CameraSensor *sensor_;
+	V4L2ControlList controls_;
+};
+
+class RkISP1ActionQueueBuffer : public FrameAction
+{
+public:
+	RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type,
+				V4L2VideoDevice *device, Buffer *buffer)
+		: FrameAction(type, frame), device_(device), buffer_(buffer)
+	{
+	}
+
+protected:
+	void run() override;
+
+private:
+	V4L2VideoDevice *device_;
+	Buffer *buffer_;
+};
+
+class RkISP1Timeline : public Timeline
+{
+public:
+	RkISP1Timeline()
+		: Timeline()
+	{
+		setDelay(SetSensor, -1, 5);
+		setDelay(SOE, 0, -1);
+		setDelay(QueueVideo, -1, 10);
+		setDelay(QueueParameters, -1, 8);
+		setDelay(QueueStatistics, -1, 8);
+	}
+
+	void bufferReady(Buffer *buffer);
+
+	void setDelay(unsigned int type, int frame, int msdelay);
+};
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_RKISP1_H__ */
diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp
new file mode 100644
index 0000000000000000..ca0cd96711d84244
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.cpp
@@ -0,0 +1,56 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.cpp - Timeline handler for Rockchip ISP1
+ */
+
+#include "rkisp1.h"
+
+#include "log.h"
+
+namespace libcamera {
+
+LOG_DECLARE_CATEGORY(RkISP1)
+
+void RkISP1ActionSetSensor::run()
+{
+	sensor_->setControls(&controls_);
+}
+
+void RkISP1ActionQueueBuffer::run()
+{
+	int ret = device_->queueBuffer(buffer_);
+	if (ret < 0)
+		LOG(RkISP1, Error) << "Failed to queue buffer";
+}
+
+void RkISP1Timeline::bufferReady(Buffer *buffer)
+{
+	/*
+	 * Calculate SOE by taking the end of DMA set by the kernel and applying
+	 * the time offsets provieded by the IPA to find the best estimate of
+	 * SOE.
+	 *
+	 * NOTE: Make sure the IPA do not set a frame offset for the SOE action
+	 * type as the frame interval is calculated using the interval between
+	 * two SOE events. So using a frame interval in the SOE estimate creates
+	 * a recursion.
+	 */
+
+	ASSERT(frameOffset(SOE) == 0);
+
+	utils::time_point soe = std::chrono::time_point<utils::clock>()
+		+ std::chrono::nanoseconds(buffer->timestamp())
+		+ timeOffset(SOE);
+
+	notifyStartOfExposure(buffer->sequence(), soe);
+}
+
+void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay)
+{
+	utils::duration delay = std::chrono::milliseconds(msdelay);
+	setRawDelay(type, frame, delay);
+}
+
+} /* namespace libcamera */