[v3,4/7] camss: Add dummy ISP class for raw only image capture

28792 diff mbox series

diff --git a/src/libcamera/pipeline/camss/camss.cpp b/src/libcamera/pipeline/camss/camss.cpp
index bcb372601..20b64ea49 100644
--- a/src/libcamera/pipeline/camss/camss.cpp
+++ b/src/libcamera/pipeline/camss/camss.cpp
@@ -38,6 +38,7 @@ 
 #include "camss_csi.h"
 #include "camss_frames.h"
 #include "camss_isp.h"
+#include "camss_isp_raw.h"
 #include "camss_isp_soft.h"
 
 namespace libcamera {
@@ -61,7 +62,10 @@  public:
 	void cancelPendingRequests();
 
 	std::unique_ptr<CamssCsiCamera> csi_;
-	std::unique_ptr<CamssIsp> isp_;
+	std::unique_ptr<CamssIsp> processedIsp_;
+	std::unique_ptr<CamssIsp> rawIsp_;
+	/* Points to either processedIsp_ or rawIsp_ */
+	CamssIsp *isp_;
 	std::unique_ptr<DelayedControls> delayedCtrls_;
 	CamssFrames frameInfos_;
 
@@ -172,23 +176,16 @@  CameraConfiguration::Status CamssCameraConfiguration::validate()
 		}
 	}
 
-	if (!processedCount) {
-		/*
-		 * \todo allow this, add dummyISP ISP class which only
-		 * calls CPU stats on ready raw output buffers + runs the result
-		 * through the softIPA to get sensor-control + metadata-info
-		 */
-		LOG(Camss, Debug)
-			<< "Camera configuration cannot support raw-only streams";
+	if (!rawCount && !processedCount)
 		return Invalid;
-	}
 
 	if (!rawCount) {
 		rawConfig.size = processedConfig.size;
 		rawConfig.bufferCount = processedConfig.bufferCount;
 	}
 
-	csiConfig_ = data_->csi_->validate(rawConfig, data_->isp_.get());
+	CamssIsp *isp = processedCount ? data_->processedIsp_.get() : data_->rawIsp_.get();
+	csiConfig_ = data_->csi_->validate(rawConfig, isp);
 	if (!csiConfig_.pixelFormat.isValid())
 		return Invalid;
 
@@ -196,7 +193,7 @@  CameraConfiguration::Status CamssCameraConfiguration::validate()
 			  << " stride " << csiConfig_.stride
 			  << " frameSize " << csiConfig_.frameSize;
 
-	ispConfig_ = data_->isp_->validate(csiConfig_, processedConfig);
+	ispConfig_ = isp->validate(csiConfig_, processedConfig);
 	if (!ispConfig_.pixelFormat.isValid())
 		return Invalid;
 
@@ -216,7 +213,7 @@  CameraConfiguration::Status CamssCameraConfiguration::validate()
 			stream = &data_->csi_->rawStream_;
 		} else {
 			hwCfg = &ispConfig_;
-			stream = &data_->isp_->outStream_;
+			stream = &isp->outStream_;
 		}
 
 		StreamConfiguration &cfg = config_[i];
@@ -321,7 +318,7 @@  PipelineHandlerCamss::generateConfiguration(Camera *camera, std::span<const Stre
 		config->addConfiguration(csiConfig);
 
 	if (processedRequested) {
-		cfg = data->isp_->generateConfiguration(csiConfig);
+		cfg = data->processedIsp_->generateConfiguration(csiConfig);
 		if (!cfg.pixelFormat.isValid())
 			return nullptr;
 
@@ -346,6 +343,11 @@  int PipelineHandlerCamss::configure(Camera *camera, CameraConfiguration *c)
 	if (ret)
 		return ret;
 
+	if (config->ispConfig_.pixelFormat == config->csiConfig_.pixelFormat)
+		data->isp_ = data->rawIsp_.get();
+	else
+		data->isp_ = data->processedIsp_.get();
+
 	return data->isp_->configure(config->csiConfig_, config->ispConfig_,
 				     &data->controlInfo_);
 }
@@ -426,7 +428,7 @@  void PipelineHandlerCamss::stopDevice(Camera *camera)
 }
 
 CamssCameraData::CamssCameraData(PipelineHandler *pipe)
-	: Camera::Private(pipe)
+	: Camera::Private(pipe), isp_(nullptr)
 {
 	frameInfos_.bufferAvailable.connect(this, &CamssCameraData::queuePendingRequests);
 }
@@ -523,27 +525,37 @@  bool PipelineHandlerCamss::match(DeviceEnumerator *enumerator)
 		data->delayedCtrls_ =
 			std::make_unique<DelayedControls>(sensor->device(), params);
 
-		data->isp_ = std::make_unique<CamssIspSoft>(this, sensor,
-							    &data->frameInfos_,
-							    &data->controlInfo_);
-		if (!data->isp_->isValid()) {
+		data->processedIsp_ = std::make_unique<CamssIspSoft>(this, sensor,
+								     &data->frameInfos_,
+								     &data->controlInfo_);
+		if (!data->processedIsp_->isValid()) {
 			LOG(Camss, Error) << "Failed to create software ISP";
 			continue;
 		}
 
-		data->isp_->inputBufferReady.connect(data->csi_.get(),
-						     &CamssCsiCamera::tryReturnBuffer);
-		data->isp_->outputBufferReady.connect(data.get(),
-						      &CamssCameraData::ispOutputBufferReady);
-		data->isp_->paramBufferReady.connect(data.get(),
-						     &CamssCameraData::ispParameterBufferReady);
-		data->isp_->statsReady.connect(data.get(), &CamssCameraData::statsReady);
-		data->isp_->metadataReady.connect(data.get(), &CamssCameraData::metadataReady);
-		data->isp_->setSensorControls.connect(data.get(), &CamssCameraData::setSensorControls);
+		data->rawIsp_ = std::make_unique<CamssIspRaw>(this, sensor,
+							      &data->frameInfos_,
+							      &data->controlInfo_);
+		if (!data->rawIsp_->isValid()) {
+			LOG(Camss, Error) << "Failed to create raw ISP";
+			continue;
+		}
+
+		for (auto isp : { data->processedIsp_.get(), data->rawIsp_.get() }) {
+			isp->inputBufferReady.connect(data->csi_.get(),
+						      &CamssCsiCamera::tryReturnBuffer);
+			isp->outputBufferReady.connect(data.get(),
+						       &CamssCameraData::ispOutputBufferReady);
+			isp->paramBufferReady.connect(data.get(),
+						      &CamssCameraData::ispParameterBufferReady);
+			isp->statsReady.connect(data.get(), &CamssCameraData::statsReady);
+			isp->metadataReady.connect(data.get(), &CamssCameraData::metadataReady);
+			isp->setSensorControls.connect(data.get(), &CamssCameraData::setSensorControls);
+		}
 
 		/* Create and register the Camera instance. */
 		std::set<Stream *> streams = {
-			&data->isp_->outStream_,
+			&data->processedIsp_->outStream_,
 			&data->csi_->rawStream_,
 		};
 		std::shared_ptr<Camera> camera =
diff --git a/src/libcamera/pipeline/camss/camss_isp_raw.cpp b/src/libcamera/pipeline/camss/camss_isp_raw.cpp
new file mode 100644
index 000000000..cd9226d7b
--- /dev/null
+++ b/src/libcamera/pipeline/camss/camss_isp_raw.cpp
@@ -0,0 +1,228 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Qualcomm CAMSS dummy ISP class for raw only image capture.
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "camss_isp_raw.h"
+
+#include <string.h>
+
+#include <libcamera/base/log.h>
+
+#include <libcamera/controls.h>
+#include <libcamera/formats.h>
+#include <libcamera/geometry.h>
+#include <libcamera/request.h>
+#include <libcamera/stream.h>
+
+#include "libcamera/internal/camera_manager.h"
+#include "libcamera/internal/camera_sensor.h"
+#include "libcamera/internal/dma_buf_allocator.h"
+#include "libcamera/internal/ipa_manager.h"
+#include "libcamera/internal/media_device.h"
+#include "libcamera/internal/pipeline_handler.h"
+#include "libcamera/internal/software_isp/debayer_params.h"
+#include "libcamera/internal/software_isp/swstats_cpu.h"
+#include "libcamera/internal/v4l2_subdevice.h"
+#include "libcamera/internal/v4l2_videodevice.h"
+
+#include "camss_frames.h"
+
+namespace libcamera {
+
+LOG_DECLARE_CATEGORY(Camss)
+
+/**
+ * \class CamssIspRaw
+ * \brief CAMSS dummy ISP class for raw only image capture
+ */
+
+/**
+ * \brief Constructs CamssIspRaw object
+ * \param[in] pipe The pipeline handler in use
+ * \param[in] sensor Pointer to the CameraSensor instance owned by the pipeline
+ * \param[in] frameInfos Pointer to CamssFrames instance owned by the pipeline
+ * \param[out] ControlInfoMap to which to add ISP provided controls
+ */
+CamssIspRaw::CamssIspRaw(PipelineHandler *pipe, const CameraSensor *sensor,
+			 CamssFrames *frameInfos, ControlInfoMap *ispControls)
+	: sensor_(sensor), frameInfos_(frameInfos)
+{
+	sharedParams_ = SharedMemObject<DebayerParams>("softIsp_params");
+	if (!sharedParams_) {
+		LOG(Camss, Error) << "Failed to create shared memory for parameters";
+		return;
+	}
+
+	const CameraManager &cm = *pipe->cameraManager();
+
+	stats_ = std::make_unique<SwStatsCpu>(cm);
+	if (!stats_->isValid()) {
+		LOG(Camss, Error) << "Failed to create SwStatsCpu object";
+		return;
+	}
+
+	stats_->statsReady.connect(this,
+				   [&](uint32_t frame, uint32_t statsBufferId) {
+					   statsReady.emit(frame, statsBufferId);
+				   });
+
+	ipa_ = pipe->createIPA<ipa::softisp::IPAProxySoftIsp>(0, 0);
+	if (!ipa_) {
+		LOG(Camss, Error) << "Creating IPA failed";
+		return;
+	}
+
+	/*
+	 * The API tuning file is made from the sensor name. If the tuning file
+	 * isn't found, fall back to the 'uncalibrated' file.
+	 */
+	std::string ipaTuningFile =
+		ipa_->configurationFile(sensor->model() + ".yaml", "uncalibrated.yaml");
+
+	IPACameraSensorInfo sensorInfo{};
+	int ret = sensor->sensorInfo(&sensorInfo);
+	if (ret) {
+		LOG(Camss, Error) << "Camera sensor information not available";
+		ipa_.reset();
+		return;
+	}
+
+	bool ccmEnabled;
+	bool lscEnabled;
+	ret = ipa_->init(IPASettings{ ipaTuningFile, sensor->model() },
+			 stats_->getStatsFD(),
+			 sharedParams_.fd(),
+			 sensorInfo,
+			 sensor->controls(),
+			 ispControls,
+			 &ccmEnabled,
+			 &lscEnabled);
+	if (ret) {
+		LOG(Camss, Error) << "IPA init failed";
+		ipa_.reset();
+		return;
+	}
+
+	ipa_->metadataReady.connect(this,
+				    [&](uint32_t frame, const ControlList &metadata) {
+					    metadataReady.emit(frame, metadata);
+				    });
+	ipa_->setSensorControls.connect(this,
+					[&](const ControlList &sensorControls) {
+						setSensorControls.emit(sensorControls);
+					});
+}
+
+CamssIspRaw::~CamssIspRaw() = default;
+
+bool CamssIspRaw::isValid()
+{
+	return !!ipa_;
+}
+
+Size CamssIspRaw::getMargins([[maybe_unused]] PixelFormat inputFormat)
+{
+	return Size();
+}
+
+StreamConfiguration CamssIspRaw::generateConfiguration(const StreamConfiguration &raw) const
+{
+	return raw;
+}
+
+StreamConfiguration CamssIspRaw::validate(const StreamConfiguration &raw,
+					  [[maybe_unused]] const StreamConfiguration &req) const
+{
+	return raw;
+}
+
+int CamssIspRaw::configure(const StreamConfiguration &inputCfg,
+			   const StreamConfiguration &outputCfg,
+			   ControlInfoMap *ispControls)
+{
+	ipa::softisp::IPAConfigInfo configInfo;
+	configInfo.sensorControls = sensor_->controls();
+
+	if (inputCfg.pixelFormat != outputCfg.pixelFormat ||
+	    inputCfg.size != outputCfg.size)
+		return -EINVAL;
+
+	int ret = ipa_->configure(configInfo, ispControls);
+	if (ret < 0)
+		return ret;
+
+	ret = stats_->configure(inputCfg);
+	if (ret < 0)
+		return ret;
+
+	/* Use 2/3 center of image to reduce CPU load */
+	Rectangle statsWindow;
+	statsWindow.width = inputCfg.size.width * 2 / 3;
+	statsWindow.height = inputCfg.size.height * 2 / 3;
+	statsWindow.x = (inputCfg.size.width - statsWindow.width) / 2;
+	statsWindow.y = (inputCfg.size.height - statsWindow.height) / 2;
+	/* stats_->setWindow() takes care of necessary alignment itself */
+	stats_->setWindow(statsWindow);
+
+	return 0;
+}
+
+int CamssIspRaw::allocateBuffers([[maybe_unused]] unsigned int bufferCount)
+{
+	frameInfos_->init({}, {});
+	return 0;
+}
+
+void CamssIspRaw::freeBuffers()
+{
+	frameInfos_->clear();
+}
+
+int CamssIspRaw::exportOutputBuffers([[maybe_unused]] const Stream *stream,
+				     [[maybe_unused]] unsigned int count,
+				     [[maybe_unused]] std::vector<std::unique_ptr<FrameBuffer>> *buffers)
+{
+	/* Must never be called when there only is a raw stream */
+	return -EINVAL;
+}
+
+void CamssIspRaw::queueBuffers(Request *request, FrameBuffer *inputBuffer)
+{
+	uint32_t frame = request->sequence();
+
+	DmaSyncer inputDmaSync(inputBuffer->planes()[0].fd, DmaSyncer::SyncType::Read);
+
+	MappedFrameBuffer input(inputBuffer, MappedFrameBuffer::MapFlag::Read);
+	if (!input.isValid()) {
+		LOG(Camss, Error) << "mmap-ing buffer(s) failed";
+		return;
+	}
+
+	ipa_->queueRequest(frame, request->controls());
+	/*
+	 * This emits statsReady which calls ipa_->processStats(), which emits
+	 * metadataReady which makes camss.cpp mark the request as completed.
+	 */
+	stats_->processFrame(frame, 0, input);
+}
+
+void CamssIspRaw::processStats(const uint32_t frame, const uint32_t statsBufferId,
+			       const ControlList &sensorControls)
+{
+	ipa_->processStats(frame, statsBufferId, sensorControls);
+}
+
+int CamssIspRaw::start()
+{
+	return ipa_->start();
+}
+
+void CamssIspRaw::stop()
+{
+	ipa_->stop();
+}
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/camss/camss_isp_raw.h b/src/libcamera/pipeline/camss/camss_isp_raw.h
new file mode 100644
index 000000000..76b9d1af8
--- /dev/null
+++ b/src/libcamera/pipeline/camss/camss_isp_raw.h
@@ -0,0 +1,72 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Qualcomm CAMSS dummy ISP class for raw only image capture.
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include <libcamera/ipa/softisp_ipa_interface.h>
+#include <libcamera/ipa/softisp_ipa_proxy.h>
+
+/* FIXME drop once using own camss IPA, which will map on IPA side */
+#include "libcamera/internal/mapped_framebuffer.h"
+#include "libcamera/internal/shared_mem_object.h"
+#include "libcamera/internal/software_isp/debayer_params.h"
+#include "libcamera/internal/v4l2_subdevice.h"
+
+#include "camss_isp.h"
+
+namespace libcamera {
+
+class CameraSensor;
+class CamssFrames;
+class ControlInfoMap;
+class DeviceEnumerator;
+class MediaDevice;
+class PipelineHandler;
+class Stream;
+class SwStatsCpu;
+class V4L2VideoDevice;
+
+class CamssIspRaw : public CamssIsp
+{
+public:
+	CamssIspRaw(PipelineHandler *pipe, const CameraSensor *sensor,
+		    CamssFrames *frameInfos, ControlInfoMap *ispControls);
+	~CamssIspRaw() override;
+
+	bool isValid() override;
+	Size getMargins(PixelFormat inputFormat) override;
+
+	StreamConfiguration generateConfiguration(const StreamConfiguration &raw) const override;
+	StreamConfiguration validate(const StreamConfiguration &raw, const StreamConfiguration &req) const override;
+	int configure(const StreamConfiguration &inputCfg,
+		      const StreamConfiguration &outputCfg,
+		      ControlInfoMap *ispControls) override;
+
+	int allocateBuffers(unsigned int bufferCount) override;
+	void freeBuffers() override;
+	int exportOutputBuffers(const Stream *stream, unsigned int count,
+				std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
+	void queueBuffers(Request *request, FrameBuffer *input) override;
+
+	void processStats(const uint32_t frame, const uint32_t statsBufferId,
+			  const ControlList &sensorControls) override;
+
+	int start() override;
+	void stop() override;
+
+private:
+	const CameraSensor *sensor_;
+	class CamssFrames *frameInfos_;
+	SharedMemObject<DebayerParams> sharedParams_;
+	std::unique_ptr<SwStatsCpu> stats_;
+	std::unique_ptr<ipa::softisp::IPAProxySoftIsp> ipa_;
+};
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/camss/meson.build b/src/libcamera/pipeline/camss/meson.build
index 2f3931dd1..518d85478 100644
--- a/src/libcamera/pipeline/camss/meson.build
+++ b/src/libcamera/pipeline/camss/meson.build
@@ -5,6 +5,7 @@  libcamera_internal_sources += files([
     'camss_csi.cpp',
     'camss_frames.cpp',
     'camss_isp.cpp',
+    'camss_isp_raw.cpp',
     'camss_isp_soft.cpp',
     'camss_util.cpp',
 ])