[v2,7/7] libcamera: ipu3: Remove IPU3Frames

19518 diff mbox series

diff --git a/src/libcamera/pipeline/ipu3/cio2.cpp b/src/libcamera/pipeline/ipu3/cio2.cpp
index 7400cb0b..e0b0d3d1 100644
--- a/src/libcamera/pipeline/ipu3/cio2.cpp
+++ b/src/libcamera/pipeline/ipu3/cio2.cpp
@@ -379,7 +379,7 @@  int CIO2Device::stop()
 	return ret;
 }
 
-FrameBuffer *CIO2Device::queueBuffer(Request *request, FrameBuffer *rawBuffer)
+FrameBuffer *CIO2Device::queueBuffer(FrameBuffer *rawBuffer)
 {
 	FrameBuffer *buffer = rawBuffer;
 
@@ -392,7 +392,6 @@  FrameBuffer *CIO2Device::queueBuffer(Request *request, FrameBuffer *rawBuffer)
 
 		buffer = availableBuffers_.front();
 		availableBuffers_.pop();
-		buffer->_d()->setRequest(request);
 	}
 
 	int ret = output_->queueBuffer(buffer);
diff --git a/src/libcamera/pipeline/ipu3/cio2.h b/src/libcamera/pipeline/ipu3/cio2.h
index bbd87eb8..42bb28b7 100644
--- a/src/libcamera/pipeline/ipu3/cio2.h
+++ b/src/libcamera/pipeline/ipu3/cio2.h
@@ -56,7 +56,7 @@  public:
 	CameraSensor *sensor() { return sensor_.get(); }
 	const CameraSensor *sensor() const { return sensor_.get(); }
 
-	FrameBuffer *queueBuffer(Request *request, FrameBuffer *rawBuffer);
+	FrameBuffer *queueBuffer(FrameBuffer *rawBuffer);
 	void tryReturnBuffer(FrameBuffer *buffer);
 	Signal<FrameBuffer *> &bufferReady() { return output_->bufferReady; }
 	Signal<uint32_t> &frameStart() { return csi2_->frameStart; }
diff --git a/src/libcamera/pipeline/ipu3/frames.cpp b/src/libcamera/pipeline/ipu3/frames.cpp
deleted file mode 100644
index a4c3477c..00000000
--- a/src/libcamera/pipeline/ipu3/frames.cpp
+++ /dev/null
@@ -1,143 +0,0 @@ 
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-/*
- * Copyright (C) 2020, Google Inc.
- *
- * frames.cpp - Intel IPU3 Frames helper
- */
-
-#include "frames.h"
-
-#include <libcamera/framebuffer.h>
-#include <libcamera/request.h>
-
-#include "libcamera/internal/framebuffer.h"
-#include "libcamera/internal/pipeline_handler.h"
-#include "libcamera/internal/v4l2_videodevice.h"
-
-namespace libcamera {
-
-LOG_DECLARE_CATEGORY(IPU3)
-
-IPU3Frames::IPU3Frames()
-{
-}
-
-void IPU3Frames::init(const std::vector<std::unique_ptr<FrameBuffer>> &paramBuffers,
-		      const std::vector<std::unique_ptr<FrameBuffer>> &statBuffers)
-{
-	for (const std::unique_ptr<FrameBuffer> &buffer : paramBuffers)
-		availableParamBuffers_.push(buffer.get());
-
-	for (const std::unique_ptr<FrameBuffer> &buffer : statBuffers)
-		availableStatBuffers_.push(buffer.get());
-
-	frameInfo_.clear();
-}
-
-void IPU3Frames::clear()
-{
-	availableParamBuffers_ = {};
-	availableStatBuffers_ = {};
-}
-
-IPU3Frames::Info *IPU3Frames::create(Request *request)
-{
-	unsigned int id = request->sequence();
-
-	if (availableParamBuffers_.empty()) {
-		LOG(IPU3, Debug) << "Parameters buffer underrun";
-		return nullptr;
-	}
-
-	if (availableStatBuffers_.empty()) {
-		LOG(IPU3, Debug) << "Statistics buffer underrun";
-		return nullptr;
-	}
-
-	FrameBuffer *paramBuffer = availableParamBuffers_.front();
-	FrameBuffer *statBuffer = availableStatBuffers_.front();
-
-	paramBuffer->_d()->setRequest(request);
-	statBuffer->_d()->setRequest(request);
-
-	availableParamBuffers_.pop();
-	availableStatBuffers_.pop();
-
-	/* \todo Remove the dynamic allocation of Info */
-	std::unique_ptr<Info> info = std::make_unique<Info>();
-
-	info->id = id;
-	info->request = request;
-	info->rawBuffer = nullptr;
-	info->paramBuffer = paramBuffer;
-	info->statBuffer = statBuffer;
-	info->paramDequeued = false;
-	info->metadataProcessed = false;
-
-	frameInfo_[id] = std::move(info);
-
-	return frameInfo_[id].get();
-}
-
-void IPU3Frames::remove(IPU3Frames::Info *info)
-{
-	/* Return params and stat buffer for reuse. */
-	availableParamBuffers_.push(info->paramBuffer);
-	availableStatBuffers_.push(info->statBuffer);
-
-	/* Delete the extended frame information. */
-	frameInfo_.erase(info->id);
-}
-
-bool IPU3Frames::tryComplete(IPU3Frames::Info *info)
-{
-	Request *request = info->request;
-
-	if (request->hasPendingBuffers())
-		return false;
-
-	if (!info->metadataProcessed)
-		return false;
-
-	if (!info->paramDequeued)
-		return false;
-
-	remove(info);
-
-	bufferAvailable.emit();
-
-	return true;
-}
-
-IPU3Frames::Info *IPU3Frames::find(unsigned int id)
-{
-	const auto &itInfo = frameInfo_.find(id);
-
-	if (itInfo != frameInfo_.end())
-		return itInfo->second.get();
-
-	LOG(IPU3, Fatal) << "Can't find tracking information for frame " << id;
-
-	return nullptr;
-}
-
-IPU3Frames::Info *IPU3Frames::find(FrameBuffer *buffer)
-{
-	for (auto const &itInfo : frameInfo_) {
-		Info *info = itInfo.second.get();
-
-		for (auto const itBuffers : info->request->buffers())
-			if (itBuffers.second == buffer)
-				return info;
-
-		if (info->rawBuffer == buffer || info->paramBuffer == buffer ||
-		    info->statBuffer == buffer)
-			return info;
-	}
-
-	LOG(IPU3, Fatal) << "Can't find tracking information from buffer";
-
-	return nullptr;
-}
-
-} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/ipu3/frames.h b/src/libcamera/pipeline/ipu3/frames.h
deleted file mode 100644
index 6e3cb915..00000000
--- a/src/libcamera/pipeline/ipu3/frames.h
+++ /dev/null
@@ -1,67 +0,0 @@ 
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-/*
- * Copyright (C) 2020, Google Inc.
- *
- * frames.h - Intel IPU3 Frames helper
- */
-
-#pragma once
-
-#include <map>
-#include <memory>
-#include <queue>
-#include <vector>
-
-#include <libcamera/base/signal.h>
-
-#include <libcamera/controls.h>
-
-namespace libcamera {
-
-class FrameBuffer;
-class IPAProxy;
-class PipelineHandler;
-class Request;
-class V4L2VideoDevice;
-struct IPABuffer;
-
-class IPU3Frames
-{
-public:
-	struct Info {
-		unsigned int id;
-		Request *request;
-
-		FrameBuffer *rawBuffer;
-		FrameBuffer *paramBuffer;
-		FrameBuffer *statBuffer;
-
-		ControlList effectiveSensorControls;
-
-		bool paramDequeued;
-		bool metadataProcessed;
-	};
-
-	IPU3Frames();
-
-	void init(const std::vector<std::unique_ptr<FrameBuffer>> &paramBuffers,
-		  const std::vector<std::unique_ptr<FrameBuffer>> &statBuffers);
-	void clear();
-
-	Info *create(Request *request);
-	void remove(Info *info);
-	bool tryComplete(Info *info);
-
-	Info *find(unsigned int id);
-	Info *find(FrameBuffer *buffer);
-
-	Signal<> bufferAvailable;
-
-private:
-	std::queue<FrameBuffer *> availableParamBuffers_;
-	std::queue<FrameBuffer *> availableStatBuffers_;
-
-	std::map<unsigned int, std::unique_ptr<Info>> frameInfo_;
-};
-
-} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp
index fa4bd0bb..a8e59dea 100644
--- a/src/libcamera/pipeline/ipu3/ipu3.cpp
+++ b/src/libcamera/pipeline/ipu3/ipu3.cpp
@@ -34,15 +34,17 @@ 
 #include "libcamera/internal/ipa_manager.h"
 #include "libcamera/internal/media_device.h"
 #include "libcamera/internal/pipeline_handler.h"
+#include "libcamera/internal/request.h"
 
 #include "cio2.h"
-#include "frames.h"
 #include "imgu.h"
 
 namespace libcamera {
 
 LOG_DEFINE_CATEGORY(IPU3)
 
+using InternalStream = Request::Private::InternalStream;
+
 static const ControlInfoMap::Map IPU3Controls = {
 	{ &controls::draft::PipelineDepth, ControlInfo(2, 3) },
 };
@@ -58,6 +60,7 @@  public:
 	int loadIPA();
 
 	void imguOutputBufferReady(FrameBuffer *buffer);
+	void imguInputBufferReady(FrameBuffer *buffer);
 	void cio2BufferReady(FrameBuffer *buffer);
 	void paramBufferReady(FrameBuffer *buffer);
 	void statBufferReady(FrameBuffer *buffer);
@@ -75,7 +78,6 @@  public:
 	Rectangle cropRegion_;
 
 	std::unique_ptr<DelayedControls> delayedCtrls_;
-	IPU3Frames frameInfos_;
 
 	std::unique_ptr<ipa::ipu3::IPAProxyIPU3> ipa_;
 
@@ -86,11 +88,14 @@  public:
 
 	ControlInfoMap ipaControls_;
 
+	std::queue<FrameBuffer *> availableParamBuffers_;
+	std::queue<FrameBuffer *> availableStatBuffers_;
 private:
 	void metadataReady(unsigned int id, const ControlList &metadata);
 	void paramsBufferReady(unsigned int id);
 	void setSensorControls(unsigned int id, const ControlList &sensorControls,
 			       const ControlList &lensControls);
+	void tryCompleteRequest(Request *request);
 };
 
 class IPU3CameraConfiguration : public CameraConfiguration
@@ -680,19 +685,17 @@  int PipelineHandlerIPU3::allocateBuffers(Camera *camera)
 	for (const std::unique_ptr<FrameBuffer> &buffer : imgu->paramBuffers_) {
 		buffer->setCookie(ipaBufferId++);
 		ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes());
+		data->availableParamBuffers_.push(buffer.get());
 	}
 
 	for (const std::unique_ptr<FrameBuffer> &buffer : imgu->statBuffers_) {
 		buffer->setCookie(ipaBufferId++);
 		ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes());
+		data->availableStatBuffers_.push(buffer.get());
 	}
 
 	data->ipa_->mapBuffers(ipaBuffers_);
 
-	data->frameInfos_.init(imgu->paramBuffers_, imgu->statBuffers_);
-	data->frameInfos_.bufferAvailable.connect(
-		data, &IPU3CameraData::queuePendingRequests);
-
 	return 0;
 }
 
@@ -700,7 +703,11 @@  int PipelineHandlerIPU3::freeBuffers(Camera *camera)
 {
 	IPU3CameraData *data = cameraData(camera);
 
-	data->frameInfos_.clear();
+	while (!data->availableStatBuffers_.empty())
+		data->availableStatBuffers_.pop();
+
+	while (!data->availableParamBuffers_.empty())
+		data->availableParamBuffers_.pop();
 
 	std::vector<unsigned int> ids;
 	for (IPABuffer &ipabuf : ipaBuffers_)
@@ -792,6 +799,12 @@  void IPU3CameraData::cancelPendingRequests()
 			pipe()->completeBuffer(request, buffer);
 		}
 
+		for (auto it : request->_d()->internalBuffers()) {
+			FrameBuffer *b = it.second;
+			b->_d()->cancel();
+			pipe()->completeBuffer(request, b);
+		}
+
 		pipe()->completeRequest(request);
 		pendingRequests_.pop();
 	}
@@ -802,31 +815,23 @@  void IPU3CameraData::queuePendingRequests()
 	while (!pendingRequests_.empty()) {
 		Request *request = pendingRequests_.front();
 
-		IPU3Frames::Info *info = frameInfos_.create(request);
-		if (!info)
-			break;
-
 		/*
 		 * Queue a buffer on the CIO2, using the raw stream buffer
 		 * provided in the request, if any, or a CIO2 internal buffer
 		 * otherwise.
 		 */
 		FrameBuffer *reqRawBuffer = request->findBuffer(&rawStream_);
-		FrameBuffer *rawBuffer = cio2_.queueBuffer(request, reqRawBuffer);
+		FrameBuffer *rawBuffer = cio2_.queueBuffer(reqRawBuffer);
 		/*
 		 * \todo If queueBuffer fails in queuing a buffer to the device,
 		 * report the request as error by cancelling the request and
 		 * calling PipelineHandler::completeRequest().
 		 */
-		if (!rawBuffer) {
-			frameInfos_.remove(info);
+		if (!rawBuffer)
 			break;
-		}
-
-		info->rawBuffer = rawBuffer;
-
-		ipa_->queueRequest(info->id, request->controls());
 
+		request->_d()->addInternalBuffer(InternalStream::Mem2Mem, rawBuffer);
+		ipa_->queueRequest(request->sequence(), request->controls());
 		pendingRequests_.pop();
 		processingRequests_.push(request);
 	}
@@ -1120,8 +1125,8 @@  int PipelineHandlerIPU3::registerCameras()
 					&IPU3CameraData::cio2BufferReady);
 		data->cio2_.bufferAvailable.connect(
 			data.get(), &IPU3CameraData::queuePendingRequests);
-		data->imgu_->input_->bufferReady.connect(&data->cio2_,
-					&CIO2Device::tryReturnBuffer);
+		data->imgu_->input_->bufferReady.connect(data.get(),
+					&IPU3CameraData::imguInputBufferReady);
 		data->imgu_->output_->bufferReady.connect(data.get(),
 					&IPU3CameraData::imguOutputBufferReady);
 		data->imgu_->viewfinder_->bufferReady.connect(data.get(),
@@ -1200,6 +1205,17 @@  int IPU3CameraData::loadIPA()
 	return 0;
 }
 
+void IPU3CameraData::tryCompleteRequest(Request *request)
+{
+	if (request->hasPendingBuffers())
+		return;
+
+	pipe()->completeRequest(request);
+
+	/* Try queue another request now that this one has completed. */
+	queuePendingRequests();
+}
+
 void IPU3CameraData::setSensorControls([[maybe_unused]] unsigned int id,
 				       const ControlList &sensorControls,
 				       const ControlList &lensControls)
@@ -1220,12 +1236,11 @@  void IPU3CameraData::setSensorControls([[maybe_unused]] unsigned int id,
 
 void IPU3CameraData::paramsBufferReady(unsigned int id)
 {
-	IPU3Frames::Info *info = frameInfos_.find(id);
-	if (!info)
-		return;
+	Request *request = queuedRequests_[id];
+	ASSERT(request);
 
 	/* Queue all buffers from the request aimed for the ImgU. */
-	for (auto it : info->request->buffers()) {
+	for (auto it : request->buffers()) {
 		const Stream *stream = it.first;
 		FrameBuffer *outbuffer = it.second;
 
@@ -1235,25 +1250,41 @@  void IPU3CameraData::paramsBufferReady(unsigned int id)
 			imgu_->viewfinder_->queueBuffer(outbuffer);
 	}
 
-	info->paramBuffer->_d()->metadata().planes()[0].bytesused =
+	FrameBuffer *paramBuffer = request->_d()->findInternalBuffer(InternalStream::Parameters);
+	paramBuffer->_d()->metadata().planes()[0].bytesused =
 		sizeof(struct ipu3_uapi_params);
-	imgu_->param_->queueBuffer(info->paramBuffer);
-	imgu_->stat_->queueBuffer(info->statBuffer);
-	imgu_->input_->queueBuffer(info->rawBuffer);
+
+	if (availableStatBuffers_.empty()) {
+		LOG(IPU3, Error) << "Statistics buffer underrun";
+		return;
+	}
+
+	FrameBuffer *statBuffer = availableStatBuffers_.front();
+	availableStatBuffers_.pop();
+	request->_d()->addInternalBuffer(InternalStream::Statistics, statBuffer);
+
+	imgu_->param_->queueBuffer(paramBuffer);
+	imgu_->stat_->queueBuffer(statBuffer);
+
+	FrameBuffer *rawBuffer = request->_d()->findInternalBuffer(InternalStream::Mem2Mem);
+	ASSERT(rawBuffer);
+
+	imgu_->input_->queueBuffer(rawBuffer);
 }
 
 void IPU3CameraData::metadataReady(unsigned int id, const ControlList &metadata)
 {
-	IPU3Frames::Info *info = frameInfos_.find(id);
-	if (!info)
-		return;
+	Request *request = queuedRequests_[id];
+	ASSERT(request);
+
+	FrameBuffer *statBuffer = request->_d()->findInternalBuffer(InternalStream::Statistics);
+	ASSERT(statBuffer);
 
-	Request *request = info->request;
 	request->metadata().merge(metadata);
+	availableStatBuffers_.push(statBuffer);
 
-	info->metadataProcessed = true;
-	if (frameInfos_.tryComplete(info))
-		pipe()->completeRequest(request);
+	pipe()->completeBuffer(request, statBuffer);
+	tryCompleteRequest(request);
 }
 
 /* -----------------------------------------------------------------------------
@@ -1268,13 +1299,7 @@  void IPU3CameraData::metadataReady(unsigned int id, const ControlList &metadata)
  */
 void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
 {
-	IPU3Frames::Info *info = frameInfos_.find(buffer);
-	if (!info)
-		return;
-
-	Request *request = info->request;
-
-	pipe()->completeBuffer(request, buffer);
+	Request *request = buffer->request();
 
 	request->metadata().set(controls::draft::PipelineDepth, 3);
 	/* \todo Actually apply the scaler crop region to the ImgU. */
@@ -1283,8 +1308,25 @@  void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
 		cropRegion_ = *scalerCrop;
 	request->metadata().set(controls::ScalerCrop, cropRegion_);
 
-	if (frameInfos_.tryComplete(info))
-		pipe()->completeRequest(request);
+	pipe()->completeBuffer(request, buffer);
+	tryCompleteRequest(request);
+}
+
+void IPU3CameraData::imguInputBufferReady(FrameBuffer *buffer)
+{
+	Request *request = buffer->request();
+
+	/*
+	 * If the Request has a Mem2Mem buffer and it's **this** buffer, then
+	 * give it back to the CIO2Device to return it to its internal queue and
+	 * flag it as completed for the purposes of the Request
+	*/
+	FrameBuffer *reqBuffer = request->_d()->findInternalBuffer(InternalStream::Mem2Mem);
+	if (reqBuffer && reqBuffer == buffer) {
+		pipe()->completeBuffer(request, buffer);
+		tryCompleteRequest(request);
+		cio2_.tryReturnBuffer(buffer);
+	}
 }
 
 /**
@@ -1296,11 +1338,7 @@  void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
  */
 void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)
 {
-	IPU3Frames::Info *info = frameInfos_.find(buffer);
-	if (!info)
-		return;
-
-	Request *request = info->request;
+	Request *request = buffer->request();
 
 	/* If the buffer is cancelled force a complete of the whole request. */
 	if (buffer->metadata().status == FrameMetadata::FrameCancelled) {
@@ -1310,7 +1348,12 @@  void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)
 			pipe()->completeBuffer(request, b);
 		}
 
-		frameInfos_.remove(info);
+		for (auto it : request->_d()->internalBuffers()) {
+			FrameBuffer *b = it.second;
+			b->_d()->cancel();
+			pipe()->completeBuffer(request, b);
+		}
+
 		pipe()->completeRequest(request);
 		return;
 	}
@@ -1324,57 +1367,51 @@  void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)
 	request->metadata().set(controls::SensorTimestamp,
 				buffer->metadata().timestamp);
 
-	info->effectiveSensorControls = delayedCtrls_->get(buffer->metadata().sequence);
-
 	if (request->findBuffer(&rawStream_))
 		pipe()->completeBuffer(request, buffer);
 
-	ipa_->fillParamsBuffer(info->id, info->paramBuffer->cookie());
+	if (availableParamBuffers_.empty()) {
+		LOG(IPU3, Error) << "Parameters buffer underrun";
+		return;
+	}
+
+	FrameBuffer *paramBuffer = availableParamBuffers_.front();
+	availableParamBuffers_.pop();
+	request->_d()->addInternalBuffer(InternalStream::Parameters, paramBuffer);
+
+	ipa_->fillParamsBuffer(request->sequence(), paramBuffer->cookie());
 }
 
 void IPU3CameraData::paramBufferReady(FrameBuffer *buffer)
 {
-	IPU3Frames::Info *info = frameInfos_.find(buffer);
-	if (!info)
-		return;
-
-	info->paramDequeued = true;
+	Request *request = buffer->request();
 
-	/*
-	 * tryComplete() will delete info if it completes the IPU3Frame.
-	 * In that event, we must have obtained the Request before hand.
-	 *
-	 * \todo Improve the FrameInfo API to avoid this type of issue
-	 */
-	Request *request = info->request;
+	pipe()->completeBuffer(request, buffer);
+	tryCompleteRequest(request);
 
-	if (frameInfos_.tryComplete(info))
-		pipe()->completeRequest(request);
+	availableParamBuffers_.push(buffer);
 }
 
 void IPU3CameraData::statBufferReady(FrameBuffer *buffer)
 {
-	IPU3Frames::Info *info = frameInfos_.find(buffer);
-	if (!info)
-		return;
-
-	Request *request = info->request;
+	Request *request = buffer->request();
 
 	if (buffer->metadata().status == FrameMetadata::FrameCancelled) {
-		info->metadataProcessed = true;
-
-		/*
-		 * tryComplete() will delete info if it completes the IPU3Frame.
-		 * In that event, we must have obtained the Request before hand.
-		 */
-		if (frameInfos_.tryComplete(info))
-			pipe()->completeRequest(request);
+		pipe()->completeBuffer(request, buffer);
+		tryCompleteRequest(request);
 
 		return;
 	}
 
-	ipa_->processStatsBuffer(info->id, request->metadata().get(controls::SensorTimestamp).value_or(0),
-				 info->statBuffer->cookie(), info->effectiveSensorControls);
+	/*
+	 * We **must** use the sequence reported by the CIO2 when fetching the
+	 * applied sensor controls.
+	 */
+	FrameBuffer *rawBuffer = request->_d()->findInternalBuffer(InternalStream::Mem2Mem);
+	ASSERT(rawBuffer);
+
+	ipa_->processStatsBuffer(request->sequence(), request->metadata().get(controls::SensorTimestamp).value_or(0),
+				 buffer->cookie(), delayedCtrls_->get(rawBuffer->metadata().sequence));
 }
 
 /*
diff --git a/src/libcamera/pipeline/ipu3/meson.build b/src/libcamera/pipeline/ipu3/meson.build
index a1b0b31a..d60e07ae 100644
--- a/src/libcamera/pipeline/ipu3/meson.build
+++ b/src/libcamera/pipeline/ipu3/meson.build
@@ -2,7 +2,6 @@ 
 
 libcamera_sources += files([
     'cio2.cpp',
-    'frames.cpp',
     'imgu.cpp',
     'ipu3.cpp',
 ])