@@ -12,7 +12,6 @@
#include <map>
#include <memory>
#include <stdint.h>
-#include <string>
#include <tuple>
#include <vector>
@@ -45,44 +44,40 @@ struct StreamConfiguration;
LOG_DECLARE_CATEGORY(SoftwareIsp)
-class SoftwareIsp : public Object
+class RawIsp : public Object
{
public:
- SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
+ RawIsp(PipelineHandler *pipe, const CameraSensor *sensor,
ControlInfoMap *ipaControls);
- ~SoftwareIsp();
+ virtual ~RawIsp();
- int loadConfiguration([[maybe_unused]] const std::string &filename) { return 0; }
+ virtual bool isValid() const;
- bool isValid() const;
-
- std::vector<PixelFormat> formats(PixelFormat input);
-
- SizeRange sizes(PixelFormat inputFormat, const Size &inputSize);
-
- std::tuple<unsigned int, unsigned int>
+ virtual std::vector<PixelFormat> formats(PixelFormat input);
+ virtual SizeRange sizes(PixelFormat inputFormat, const Size &inputSize);
+ virtual std::tuple<unsigned int, unsigned int>
strideAndFrameSize(const PixelFormat &outputFormat, const Size &size);
- uint32_t preferredInputStride(const PixelFormat &inputFormat, const Size &size);
+ virtual uint32_t preferredInputStride(const PixelFormat &inputFormat, const Size &size);
- int configure(const StreamConfiguration &inputCfg,
- const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
- const ipa::softisp::IPAConfigInfo &configInfo,
- ControlInfoMap *ipaControls);
+ virtual int configure(const StreamConfiguration &inputCfg,
+ const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
+ const ipa::softisp::IPAConfigInfo &configInfo,
+ ControlInfoMap *ipaControls);
- int exportBuffers(const Stream *stream, unsigned int count,
- std::vector<std::unique_ptr<FrameBuffer>> *buffers);
+ virtual int exportBuffers(const Stream *stream, unsigned int count,
+ std::vector<std::unique_ptr<FrameBuffer>> *buffers);
- void processStats(const uint32_t frame, const uint32_t bufferId,
- const ControlList &sensorControls);
+ virtual void processStats(const uint32_t frame, const uint32_t bufferId,
+ const ControlList &sensorControls);
- int start();
- void stop();
+ virtual int start();
+ virtual void stop();
- void queueRequest(const uint32_t frame, const ControlList &controls);
- int queueBuffers(uint32_t frame, FrameBuffer *input,
- const std::map<const Stream *, FrameBuffer *> &outputs);
+ virtual void queueRequest(const uint32_t frame, const ControlList &controls);
+ virtual int queueBuffers(uint32_t frame, FrameBuffer *input,
+ const std::map<const Stream *, FrameBuffer *> &outputs);
- void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output);
+ virtual void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output);
Signal<FrameBuffer *> inputBufferReady;
Signal<FrameBuffer *> outputBufferReady;
@@ -90,14 +85,16 @@ public:
Signal<uint32_t, const ControlList &> metadataReady;
Signal<const ControlList &> setSensorControls;
-private:
- void saveIspParams(uint32_t frame);
- void setSensorCtrls(const ControlList &sensorControls);
- void statsReady(uint32_t frame, uint32_t bufferId);
+protected:
+ virtual void saveIspParams(uint32_t frame);
+ virtual void setSensorCtrls(const ControlList &sensorControls);
+ virtual void statsReady(uint32_t frame, uint32_t bufferId);
void inputReady(FrameBuffer *input);
void outputReady(FrameBuffer *output);
- std::unique_ptr<Debayer> debayer_;
+
Thread ispWorkerThread_;
+ bool initialized_ = false;
+ unsigned int inputStride_ = 0;
SharedMemObject<DebayerParams> sharedParams_;
DebayerParams debayerParams_;
DmaBufAllocator dmaHeap_;
@@ -108,6 +105,47 @@ private:
std::unique_ptr<ipa::softisp::IPAProxySoftIsp> ipa_;
std::deque<FrameBuffer *> queuedInputBuffers_;
std::deque<FrameBuffer *> queuedOutputBuffers_;
+
+private:
+ int init(PipelineHandler *pipe, const CameraSensor *sensor,
+ ControlInfoMap *ipaControls);
+};
+
+class DebayerIsp : public RawIsp
+{
+public:
+ DebayerIsp(PipelineHandler *pipe, const CameraSensor *sensor,
+ ControlInfoMap *ipaControls);
+ ~DebayerIsp();
+
+ bool isValid() const override;
+
+ std::vector<PixelFormat> formats(PixelFormat input) override;
+ SizeRange sizes(PixelFormat inputFormat, const Size &inputSize) override;
+ std::tuple<unsigned int, unsigned int>
+ strideAndFrameSize(const PixelFormat &outputFormat, const Size &size) override;
+ uint32_t preferredInputStride(const PixelFormat &inputFormat, const Size &size) override;
+
+ int configure(const StreamConfiguration &inputCfg,
+ const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
+ const ipa::softisp::IPAConfigInfo &configInfo,
+ ControlInfoMap *ipaControls) override;
+
+ int exportBuffers(const Stream *stream, unsigned int count,
+ std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
+
+ virtual int start() override;
+ virtual void stop() override;
+
+ int queueBuffers(uint32_t frame, FrameBuffer *input,
+ const std::map<const Stream *, FrameBuffer *> &outputs) override;
+
+ void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output) override;
+
+private:
+ int init(PipelineHandler *pipe);
+
+ std::unique_ptr<Debayer> debayer_;
};
} /* namespace libcamera */
@@ -359,7 +359,9 @@ public:
bool useConversion_;
std::unique_ptr<Converter> converter_;
- std::unique_ptr<SoftwareIsp> swIsp_;
+ std::shared_ptr<RawIsp> rawIsp_;
+ std::shared_ptr<DebayerIsp> debayerIsp_;
+ std::shared_ptr<RawIsp> swIsp_;
SimpleFrames frameInfo_;
Rectangle scalerCrop_;
@@ -618,17 +620,30 @@ int SimpleCameraData::init()
* Instantiate Soft ISP if this is enabled for the given driver and no converter is used.
*/
if (!converter_ && pipe->swIspEnabled()) {
- swIsp_ = std::make_unique<SoftwareIsp>(pipe, sensor_.get(), &controlInfo_);
- if (!swIsp_->isValid()) {
+ rawIsp_ = std::make_unique<RawIsp>(pipe, sensor_.get(), &controlInfo_);
+ if (!rawIsp_->isValid()) {
LOG(SimplePipeline, Warning)
<< "Failed to create software ISP, disabling software debayering";
- swIsp_.reset();
+ rawIsp_.reset();
} else {
- swIsp_->inputBufferReady.connect(this, &SimpleCameraData::conversionInputDone);
- swIsp_->outputBufferReady.connect(this, &SimpleCameraData::conversionOutputDone);
- swIsp_->ispStatsReady.connect(this, &SimpleCameraData::ispStatsReady);
- swIsp_->metadataReady.connect(this, &SimpleCameraData::metadataReady);
- swIsp_->setSensorControls.connect(this, &SimpleCameraData::setSensorControls);
+ rawIsp_->inputBufferReady.connect(this, &SimpleCameraData::conversionInputDone);
+ rawIsp_->outputBufferReady.connect(this, &SimpleCameraData::conversionOutputDone);
+ rawIsp_->ispStatsReady.connect(this, &SimpleCameraData::ispStatsReady);
+ rawIsp_->metadataReady.connect(this, &SimpleCameraData::metadataReady);
+ rawIsp_->setSensorControls.connect(this, &SimpleCameraData::setSensorControls);
+ }
+
+ debayerIsp_ = std::make_unique<DebayerIsp>(pipe, sensor_.get(), &controlInfo_);
+ if (!debayerIsp_->isValid()) {
+ LOG(SimplePipeline, Warning)
+ << "Failed to create software ISP, disabling software debayering";
+ debayerIsp_.reset();
+ } else {
+ debayerIsp_->inputBufferReady.connect(this, &SimpleCameraData::conversionInputDone);
+ debayerIsp_->outputBufferReady.connect(this, &SimpleCameraData::conversionOutputDone);
+ debayerIsp_->ispStatsReady.connect(this, &SimpleCameraData::ispStatsReady);
+ debayerIsp_->metadataReady.connect(this, &SimpleCameraData::metadataReady);
+ debayerIsp_->setSensorControls.connect(this, &SimpleCameraData::setSensorControls);
}
}
@@ -744,9 +759,9 @@ void SimpleCameraData::tryPipeline(unsigned int code, const Size &size)
if (converter_) {
config.outputFormats = converter_->formats(pixelFormat);
config.outputSizes = converter_->sizes(format.size);
- } else if (swIsp_) {
- config.outputFormats = swIsp_->formats(pixelFormat);
- config.outputSizes = swIsp_->sizes(pixelFormat, format.size);
+ } else if (debayerIsp_) {
+ config.outputFormats = debayerIsp_->formats(pixelFormat);
+ config.outputSizes = debayerIsp_->sizes(pixelFormat, format.size);
if (config.outputFormats.empty()) {
/* Do not use swIsp for unsupported pixelFormat's. */
config.outputFormats = { pixelFormat };
@@ -888,7 +903,7 @@ void SimpleCameraData::imageBufferReady(FrameBuffer *buffer)
* point converting an erroneous buffer.
*/
if (buffer->metadata().status != FrameMetadata::FrameSuccess) {
- if (!useConversion_ || rawStream_) {
+ if (!useConversion_) {
/* No conversion, just complete the request. */
Request *request = buffer->request();
pipe->completeBuffer(request, buffer);
@@ -904,13 +919,16 @@ void SimpleCameraData::imageBufferReady(FrameBuffer *buffer)
* buffer for capture (unless the stream is being stopped), and
* complete the request with all the user-facing buffers.
*/
- if (buffer->metadata().status != FrameMetadata::FrameCancelled)
+ if (!rawStream_ &&
+ buffer->metadata().status != FrameMetadata::FrameCancelled)
video_->queueBuffer(buffer);
if (conversionQueue_.empty())
return;
const RequestOutputs &outputs = conversionQueue_.front();
+ if (rawStream_)
+ pipe->completeBuffer(outputs.request, buffer);
for (auto &[stream, buf] : outputs.outputs)
pipe->completeBuffer(outputs.request, buf);
SimpleFrameInfo *info = frameInfo_.find(outputs.request->sequence());
@@ -1295,7 +1313,14 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate()
* require any conversion, similar to raw capture use cases). This is
* left as a future improvement.
*/
- needConversion_ = config_.size() > 1 + rawCount;
+ needConversion_ = (config_.size() > 1 + rawCount) || data_->rawIsp_;
+
+ data_->swIsp_ = data_->rawIsp_;
+ for (unsigned int i = 0; i < config_.size(); ++i)
+ if (!isRaw(config_[i])) {
+ data_->swIsp_ = data_->debayerIsp_;
+ break;
+ }
for (unsigned int i = 0; i < config_.size(); ++i) {
StreamConfiguration &cfg = config_[i];
@@ -1637,8 +1662,8 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c)
if (rawStream)
data->rawStream_ = &data->streams_[i];
}
-
- if (outputCfgs.empty())
+ if (outputCfgs.empty() &&
+ (data->converter_ || !data->rawStream_))
return 0;
StreamConfiguration inputCfg;
@@ -1787,7 +1812,7 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request)
int ret;
std::map<const Stream *, FrameBuffer *> buffers;
- bool metadataRequired = false;
+ bool metadataRequired = data->useConversion_ && !!data->swIsp_;
for (auto &[stream, buffer] : request->buffers()) {
/*
@@ -1797,7 +1822,6 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request)
*/
if (data->useConversion_ && stream != data->rawStream_) {
buffers.emplace(stream, buffer);
- metadataRequired = !!data->swIsp_;
} else {
ret = data->video_->queueBuffer(buffer);
if (ret < 0)
@@ -1811,7 +1835,6 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request)
if (data->swIsp_)
data->swIsp_->queueRequest(request->sequence(), request->controls());
}
-
return 0;
}
@@ -25,6 +25,7 @@
#include "libcamera/internal/bayer_format.h"
#include "libcamera/internal/framebuffer.h"
+#include "libcamera/internal/mapped_framebuffer.h"
#include "libcamera/internal/software_isp/debayer_params.h"
#include "debayer_cpu.h"
@@ -42,58 +43,42 @@ namespace libcamera {
LOG_DEFINE_CATEGORY(SoftwareIsp)
/**
- * \class SoftwareIsp
- * \brief Class for the Software ISP
+ * \class RawIsp
+ * \brief Base class for software image processing
*/
/**
- * \var SoftwareIsp::inputBufferReady
- * \brief A signal emitted when the input frame buffer completes
- */
-
-/**
- * \var SoftwareIsp::outputBufferReady
- * \brief A signal emitted when the output frame buffer completes
- */
-
-/**
- * \var SoftwareIsp::ispStatsReady
- * \brief A signal emitted when the statistics for IPA are ready
- */
-
-/**
- * \var SoftwareIsp::metadataReady
- * \brief A signal emitted when the metadata for IPA is ready
- */
-
-/**
- * \var SoftwareIsp::setSensorControls
- * \brief A signal emitted when the values to write to the sensor controls are
- * ready
- */
-
-/**
- * \brief Constructs SoftwareIsp object
+ * \brief Constructs RawIsp object
* \param[in] pipe The pipeline handler in use
* \param[in] sensor Pointer to the CameraSensor instance owned by the pipeline handler
* \param[out] ipaControls The IPA controls to update
*/
-SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
+RawIsp::RawIsp(PipelineHandler *pipe, const CameraSensor *sensor,
ControlInfoMap *ipaControls)
: ispWorkerThread_("SWIspWorker"),
dmaHeap_(DmaBufAllocator::DmaBufAllocatorFlag::CmaHeap |
DmaBufAllocator::DmaBufAllocatorFlag::SystemHeap |
DmaBufAllocator::DmaBufAllocatorFlag::UDmaBuf)
+{
+ initialized_ = init(pipe, sensor, ipaControls) == 0;
+}
+
+RawIsp::~RawIsp()
+{
+}
+
+int RawIsp::init(PipelineHandler *pipe, const CameraSensor *sensor,
+ ControlInfoMap *ipaControls)
{
if (!dmaHeap_.isValid()) {
LOG(SoftwareIsp, Error) << "Failed to create DmaBufAllocator object";
- return;
+ return -1;
}
sharedParams_ = SharedMemObject<DebayerParams>("softIsp_params");
if (!sharedParams_) {
LOG(SoftwareIsp, Error) << "Failed to create shared memory for parameters";
- return;
+ return -1;
}
const CameraManager &cm = *pipe->cameraManager();
@@ -101,46 +86,15 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
stats_ = std::make_shared<SwStatsCpu>(cm);
if (!stats_->isValid()) {
LOG(SoftwareIsp, Error) << "Failed to create SwStatsCpu object";
- return;
- }
- stats_->statsReady.connect(this, &SoftwareIsp::statsReady);
-
-#if HAVE_DEBAYER_EGL
- const GlobalConfiguration &configuration = cm._d()->configuration();
- std::optional<std::string> softISPMode = configuration.option<std::string>({ "software_isp", "mode" });
- if (softISPMode) {
- if (softISPMode != "gpu" && softISPMode != "cpu") {
- LOG(SoftwareIsp, Error)
- << "Invalid software ISP mode \""
- << softISPMode.value()
- << "\", must be \"cpu\" or \"gpu\"";
- return;
- }
- }
-
- if (!softISPMode || softISPMode == "gpu") {
- auto display = eGL::probeDisplay();
- if (display != EGL_NO_DISPLAY) {
- debayer_ = std::make_unique<DebayerEGL>(stats_, cm, display);
- } else {
- LOG(SoftwareIsp, Info)
- << "EGL not available, falling back to CPU debayer";
- }
+ return -1;
}
-
-#endif
- if (!debayer_)
- debayer_ = std::make_unique<DebayerCpu>(stats_, cm);
-
- debayer_->inputBufferReady.connect(this, &SoftwareIsp::inputReady);
- debayer_->outputBufferReady.connect(this, &SoftwareIsp::outputReady);
+ stats_->statsReady.connect(this, &RawIsp::statsReady);
ipa_ = pipe->createIPA<ipa::softisp::IPAProxySoftIsp>(0, 0);
if (!ipa_) {
LOG(SoftwareIsp, Error)
<< "Creating IPA for software ISP failed";
- debayer_.reset();
- return;
+ return -1;
}
/*
@@ -154,7 +108,7 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
int ret = sensor->sensorInfo(&sensorInfo);
if (ret) {
LOG(SoftwareIsp, Error) << "Camera sensor information not available";
- return;
+ return -1;
}
ret = ipa_->init(IPASettings{ ipaTuningFile, sensor->model() },
@@ -167,46 +121,29 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
&lscEnabled_);
if (ret) {
LOG(SoftwareIsp, Error) << "IPA init failed";
- debayer_.reset();
- return;
+ return -1;
}
- ipa_->paramsComputed.connect(this, &SoftwareIsp::saveIspParams);
+ ipa_->paramsComputed.connect(this, &RawIsp::saveIspParams);
ipa_->metadataReady.connect(this,
[this](uint32_t frame, const ControlList &metadata) {
metadataReady.emit(frame, metadata);
});
- ipa_->setSensorControls.connect(this, &SoftwareIsp::setSensorCtrls);
+ ipa_->setSensorControls.connect(this, &RawIsp::setSensorCtrls);
- debayer_->moveToThread(&ispWorkerThread_);
-}
-
-SoftwareIsp::~SoftwareIsp()
-{
- /* make sure to destroy the DebayerCpu before the ispWorkerThread_ is gone */
- debayer_.reset();
+ return 0;
}
-/**
- * \fn int SoftwareIsp::loadConfiguration([[maybe_unused]] const std::string &filename)
- * \brief Load a configuration from a file
- * \param[in] filename The file to load the configuration data from
- *
- * Currently is a stub doing nothing and always returning "success".
- *
- * \return 0 on success
- */
-
/**
* \brief Process the statistics gathered
* \param[in] frame The frame number
* \param[in] bufferId ID of the statistics buffer
* \param[in] sensorControls The sensor controls
*
- * Requests the IPA to calculate new parameters for ISP and new control
+ * Requests the IPA to calculate new parameters for software ISP and new control
* values for the sensor.
*/
-void SoftwareIsp::processStats(const uint32_t frame, const uint32_t bufferId,
+void RawIsp::processStats(const uint32_t frame, const uint32_t bufferId,
const ControlList &sensorControls)
{
ASSERT(ipa_);
@@ -214,12 +151,12 @@ void SoftwareIsp::processStats(const uint32_t frame, const uint32_t bufferId,
}
/**
- * \brief Check the validity of Software Isp object
- * \return True if Software Isp is valid, false otherwise
+ * \brief Check the validity of the Isp object
+ * \return True if the Isp is valid, false otherwise
*/
-bool SoftwareIsp::isValid() const
+bool RawIsp::isValid() const
{
- return !!debayer_;
+ return initialized_;
}
/**
@@ -227,11 +164,9 @@ bool SoftwareIsp::isValid() const
* \param[in] inputFormat The input format
* \return All the supported output formats or an empty vector if there are none
*/
-std::vector<PixelFormat> SoftwareIsp::formats(PixelFormat inputFormat)
+std::vector<PixelFormat> RawIsp::formats(PixelFormat inputFormat)
{
- ASSERT(debayer_);
-
- return debayer_->formats(inputFormat);
+ return std::vector<PixelFormat>({ inputFormat });
}
/**
@@ -240,11 +175,9 @@ std::vector<PixelFormat> SoftwareIsp::formats(PixelFormat inputFormat)
* \param[in] inputSize The input frame size
* \return The valid size range or an empty range if there are none
*/
-SizeRange SoftwareIsp::sizes(PixelFormat inputFormat, const Size &inputSize)
+SizeRange RawIsp::sizes([[maybe_unused]] PixelFormat inputFormat, const Size &inputSize)
{
- ASSERT(debayer_);
-
- return debayer_->sizes(inputFormat, inputSize);
+ return SizeRange(inputSize);
}
/**
@@ -255,11 +188,9 @@ SizeRange SoftwareIsp::sizes(PixelFormat inputFormat, const Size &inputSize)
* if there is no valid output config
*/
std::tuple<unsigned int, unsigned int>
-SoftwareIsp::strideAndFrameSize(const PixelFormat &outputFormat, const Size &size)
+RawIsp::strideAndFrameSize([[maybe_unused]] const PixelFormat &outputFormat, const Size &size)
{
- ASSERT(debayer_);
-
- return debayer_->strideAndFrameSize(outputFormat, size);
+ return std::make_tuple(inputStride_, inputStride_ * size.height);
}
/**
@@ -268,33 +199,34 @@ SoftwareIsp::strideAndFrameSize(const PixelFormat &outputFormat, const Size &siz
* \param[in] size The input size (width and height in pixels)
* \return The preferred input stride in bytes or 0 if there is no preference
*/
-uint32_t SoftwareIsp::preferredInputStride(const PixelFormat &inputFormat, const Size &size)
+uint32_t RawIsp::preferredInputStride([[maybe_unused]] const PixelFormat &inputFormat, [[maybe_unused]] const Size &size)
{
- ASSERT(debayer_);
-
- return debayer_->preferredInputStride(inputFormat, size);
+ return 0;
}
/**
- * \brief Configure the SoftwareIsp object according to the passed in parameters
+ * \brief Configure the RawIsp object according to the passed in parameters
* \param[in] inputCfg The input configuration
* \param[in] outputCfgs The output configurations
* \param[in] configInfo The IPA configuration data, received from the pipeline handler
* \param[out] ipaControls The IPA controls to update
* \return 0 on success, a negative errno on failure
*/
-int SoftwareIsp::configure(const StreamConfiguration &inputCfg,
- const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
- const ipa::softisp::IPAConfigInfo &configInfo,
- ControlInfoMap *ipaControls)
+int RawIsp::configure(const StreamConfiguration &inputCfg,
+ [[maybe_unused]] const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
+ const ipa::softisp::IPAConfigInfo &configInfo,
+ ControlInfoMap *ipaControls)
{
- ASSERT(ipa_ && debayer_);
+ ASSERT(ipa_);
- int ret = ipa_->configure(configInfo, ipaControls);
+ inputStride_ = inputCfg.stride;
+
+ int ret = stats_->configure(inputCfg);
if (ret < 0)
return ret;
+ stats_->setWindow(Rectangle(inputCfg.size));
- ret = debayer_->configure(inputCfg, outputCfgs, ccmEnabled_, lscEnabled_);
+ ret = ipa_->configure(configInfo, ipaControls);
if (ret < 0)
return ret;
@@ -307,23 +239,18 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg,
}
/**
- * \brief Export the buffers from the Software ISP
+ * \brief Export the buffers from the software ISP
* \param[in] stream Output stream exporting the buffers
* \param[in] count Number of buffers to allocate
* \param[out] buffers Vector to store the allocated buffers
* \return The number of allocated buffers on success or a negative error code
* otherwise
*/
-int SoftwareIsp::exportBuffers(const Stream *stream, unsigned int count,
- std::vector<std::unique_ptr<FrameBuffer>> *buffers)
+int RawIsp::exportBuffers([[maybe_unused]] const Stream *stream, [[maybe_unused]] unsigned int count,
+ [[maybe_unused]] std::vector<std::unique_ptr<FrameBuffer>> *buffers)
{
- ASSERT(debayer_ != nullptr);
-
- /* single output for now */
- if (stream == nullptr)
- return -EINVAL;
-
- return dmaHeap_.exportBuffers(count, { debayer_->frameSize() }, buffers);
+ /* There are no separate output buffers for raw streams. */
+ return -1;
}
/**
@@ -331,53 +258,37 @@ int SoftwareIsp::exportBuffers(const Stream *stream, unsigned int count,
* \param[in] frame The number of the frame which will be processed next
* \param[in] controls The controls for the \a frame
*/
-void SoftwareIsp::queueRequest(const uint32_t frame, const ControlList &controls)
+void RawIsp::queueRequest(const uint32_t frame, const ControlList &controls)
{
ipa_->queueRequest(frame, controls);
}
/**
- * \brief Queue buffers to Software ISP
+ * \brief Queue buffers to the software ISP
* \param[in] frame The frame number
* \param[in] input The input framebuffer
* \param[in] outputs The container holding the output stream pointers and
* their respective frame buffer outputs
* \return 0 on success, a negative errno on failure
*/
-int SoftwareIsp::queueBuffers(uint32_t frame, FrameBuffer *input,
- const std::map<const Stream *, FrameBuffer *> &outputs)
+int RawIsp::queueBuffers(uint32_t frame, FrameBuffer *input,
+ const std::map<const Stream *, FrameBuffer *> &outputs)
{
- /*
- * Validate the outputs as a sanity check: at least one output is
- * required, all outputs must reference a valid stream.
- */
- if (outputs.empty())
- return -EINVAL;
-
- /* We only support a single stream for now. */
- if (outputs.size() != 1)
+ if (outputs.size() > 0)
return -EINVAL;
- for (auto [stream, buffer] : outputs) {
- if (!buffer)
- return -EINVAL;
- }
-
queuedInputBuffers_.push_back(input);
- for (const auto &[stream, buffer] : outputs) {
- queuedOutputBuffers_.push_back(buffer);
- process(frame, input, buffer);
- }
+ process(frame, input, nullptr);
return 0;
}
/**
- * \brief Starts the Software ISP streaming operation
+ * \brief Starts the software ISP streaming operation
* \return 0 on success, any other value indicates an error
*/
-int SoftwareIsp::start()
+int RawIsp::start()
{
int ret = ipa_->start();
if (ret)
@@ -385,8 +296,7 @@ int SoftwareIsp::start()
ispWorkerThread_.start();
- return debayer_->invokeMethod(&Debayer::start,
- ConnectionTypeBlocking);
+ return 0;
}
/**
@@ -397,14 +307,11 @@ int SoftwareIsp::start()
*
* This function will not take any action when the ISP is not already running.
*/
-void SoftwareIsp::stop()
+void RawIsp::stop()
{
if (!ispWorkerThread_.isRunning())
return;
- debayer_->invokeMethod(&Debayer::stop,
- ConnectionTypeBlocking);
-
ispWorkerThread_.exit();
ispWorkerThread_.wait();
@@ -426,45 +333,342 @@ void SoftwareIsp::stop()
}
/**
- * \brief Passes the input framebuffer to the ISP worker to process
+ * \brief Passes the input framebuffer to the software ISP worker to process
* \param[in] frame The frame number
* \param[in] input The input framebuffer
- * \param[out] output The framebuffer to write the processed frame to
+ * \param[out] output The framebuffer to write the processed frame to; if
+ * nullptr then debayering is skipped and stats are processed synchronously
*/
-void SoftwareIsp::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output)
+void RawIsp::process(uint32_t frame, FrameBuffer *input, [[maybe_unused]] FrameBuffer *output)
{
ipa_->computeParams(frame);
- debayer_->invokeMethod(&Debayer::process,
- ConnectionTypeQueued, frame, input, output, debayerParams_);
+
+ /* Compute stats, which are otherwise computed in debayering */
+ if (frame % SwStatsCpu::kStatPerNumFrames) {
+ stats_->finishFrame(frame, 0);
+ } else {
+ MappedFrameBuffer inputMapped(input, MappedFrameBuffer::MapFlag::Read);
+ stats_->processFrame(frame, 0, inputMapped);
+ }
+
+ outputReady(output);
+ inputReady(input);
}
-void SoftwareIsp::saveIspParams([[maybe_unused]] uint32_t frame)
+/**
+ * \brief TODO
+ */
+void RawIsp::saveIspParams([[maybe_unused]] uint32_t frame)
{
debayerParams_ = *sharedParams_;
}
-void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls)
+/**
+ * \brief TODO
+ */
+void RawIsp::setSensorCtrls(const ControlList &sensorControls)
{
setSensorControls.emit(sensorControls);
}
-void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId)
+/**
+ * \brief TODO
+ */
+void RawIsp::statsReady(uint32_t frame, uint32_t bufferId)
{
ispStatsReady.emit(frame, bufferId);
}
-void SoftwareIsp::inputReady(FrameBuffer *input)
+/**
+ * \brief TODO
+ */
+void RawIsp::inputReady(FrameBuffer *input)
{
ASSERT(queuedInputBuffers_.front() == input);
queuedInputBuffers_.pop_front();
inputBufferReady.emit(input);
}
-void SoftwareIsp::outputReady(FrameBuffer *output)
+/**
+ * \brief TODO
+ */
+void RawIsp::outputReady(FrameBuffer *output)
{
+ if (output == nullptr)
+ return;
+
ASSERT(queuedOutputBuffers_.front() == output);
queuedOutputBuffers_.pop_front();
outputBufferReady.emit(output);
}
+/**
+ * \var RawIsp::inputBufferReady
+ * \brief A signal emitted when the input frame buffer completes
+ */
+
+/**
+ * \var RawIsp::outputBufferReady
+ * \brief A signal emitted when the output frame buffer completes
+ */
+
+/**
+ * \var RawIsp::ispStatsReady
+ * \brief A signal emitted when the statistics for IPA are ready
+ */
+
+/**
+ * \var RawIsp::metadataReady
+ * \brief A signal emitted when the metadata for IPA is ready
+ */
+
+/**
+ * \var RawIsp::setSensorControls
+ * \brief A signal emitted when the values to write to the sensor controls are
+ * ready
+ */
+
+/**
+ * \class DebayerIsp
+ * \brief Debayering software ISP class
+ */
+
+/**
+ * \var RawIsp::ispWorkerThread_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::initialized_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::inputStride_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::sharedParams_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::debayerParams_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::dmaHeap_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::ccmEnabled_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::lscEnabled_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::stats_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::ipa_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::queuedInputBuffers_
+ * \brief TODO
+ */
+
+/**
+ * \var RawIsp::queuedOutputBuffers_
+ * \brief TODO
+ */
+
+/**
+ * \brief Constructs DebayerIsp object
+ * \param[in] pipe The pipeline handler in use
+ * \param[in] sensor Pointer to the CameraSensor instance owned by the pipeline handler
+ * \param[out] ipaControls The IPA controls to update
+ */
+DebayerIsp::DebayerIsp(PipelineHandler *pipe, const CameraSensor *sensor,
+ ControlInfoMap *ipaControls)
+ : RawIsp(pipe, sensor, ipaControls)
+{
+ if (RawIsp::isValid())
+ init(pipe);
+}
+
+DebayerIsp::~DebayerIsp()
+{
+ /* make sure to destroy the DebayerCpu before the ispWorkerThread_ is gone */
+ debayer_.reset();
+}
+
+int DebayerIsp::init(PipelineHandler *pipe)
+{
+ const CameraManager &cm = *pipe->cameraManager();
+
+#if HAVE_DEBAYER_EGL
+ const GlobalConfiguration &configuration = cm._d()->configuration();
+ std::optional<std::string> softISPMode = configuration.option<std::string>({ "software_isp", "mode" });
+ if (softISPMode) {
+ if (softISPMode != "gpu" && softISPMode != "cpu") {
+ LOG(SoftwareIsp, Error)
+ << "Invalid software ISP mode \""
+ << softISPMode.value()
+ << "\", must be \"cpu\" or \"gpu\"";
+ return -1;
+ }
+ }
+
+ if (!softISPMode || softISPMode == "gpu") {
+ auto display = eGL::probeDisplay();
+ if (display != EGL_NO_DISPLAY) {
+ debayer_ = std::make_unique<DebayerEGL>(stats_, cm, display);
+ } else {
+ LOG(SoftwareIsp, Info)
+ << "EGL not available, falling back to CPU debayer";
+ }
+ }
+
+#endif
+ if (!debayer_)
+ debayer_ = std::make_unique<DebayerCpu>(stats_, cm);
+
+ debayer_->inputBufferReady.connect<RawIsp>(this, &DebayerIsp::inputReady);
+ debayer_->outputBufferReady.connect<RawIsp>(this, &DebayerIsp::outputReady);
+ debayer_->moveToThread(&ispWorkerThread_);
+
+ return 0;
+}
+
+bool DebayerIsp::isValid() const
+{
+ return RawIsp::isValid() && !!debayer_;
+}
+
+std::vector<PixelFormat> DebayerIsp::formats(PixelFormat inputFormat)
+{
+ ASSERT(debayer_);
+
+ return debayer_->formats(inputFormat);
+}
+
+SizeRange DebayerIsp::sizes(PixelFormat inputFormat, const Size &inputSize)
+{
+ ASSERT(debayer_);
+
+ return debayer_->sizes(inputFormat, inputSize);
+}
+
+std::tuple<unsigned int, unsigned int>
+DebayerIsp::strideAndFrameSize(const PixelFormat &outputFormat, const Size &size)
+{
+ ASSERT(debayer_);
+
+ return debayer_->strideAndFrameSize(outputFormat, size);
+}
+
+uint32_t DebayerIsp::preferredInputStride(const PixelFormat &inputFormat, const Size &size)
+{
+ ASSERT(debayer_);
+
+ return debayer_->preferredInputStride(inputFormat, size);
+}
+
+int DebayerIsp::configure(const StreamConfiguration &inputCfg,
+ const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
+ const ipa::softisp::IPAConfigInfo &configInfo,
+ ControlInfoMap *ipaControls)
+{
+ ASSERT(ipa_ && debayer_);
+
+ int ret = ipa_->configure(configInfo, ipaControls);
+ if (ret < 0)
+ return ret;
+
+ ret = debayer_->configure(inputCfg, outputCfgs, ccmEnabled_, lscEnabled_);
+ if (ret < 0)
+ return ret;
+
+ LOG(SoftwareIsp, Info)
+ << "Input " << inputCfg.size
+ << "-" << BayerFormat::fromPixelFormat(inputCfg.pixelFormat)
+ << " stride " << inputCfg.stride;
+
+ return 0;
+}
+
+int DebayerIsp::exportBuffers(const Stream *stream, unsigned int count,
+ std::vector<std::unique_ptr<FrameBuffer>> *buffers)
+{
+ ASSERT(debayer_ != nullptr);
+
+ /* single output for now */
+ if (stream == nullptr)
+ return -EINVAL;
+
+ return dmaHeap_.exportBuffers(count, { debayer_->frameSize() }, buffers);
+}
+
+int DebayerIsp::queueBuffers(uint32_t frame, FrameBuffer *input,
+ const std::map<const Stream *, FrameBuffer *> &outputs)
+{
+ /* We only support a single stream for now. */
+ if (outputs.size() != 1)
+ return -EINVAL;
+
+ for (auto [stream, buffer] : outputs) {
+ if (!buffer)
+ return -EINVAL;
+ }
+
+ queuedInputBuffers_.push_back(input);
+
+ for (const auto &[stream, buffer] : outputs) {
+ queuedOutputBuffers_.push_back(buffer);
+ process(frame, input, buffer);
+ }
+
+ return 0;
+}
+
+int DebayerIsp::start()
+{
+ int ret = RawIsp::start();
+ if (ret)
+ return ret;
+
+ return debayer_->invokeMethod(&Debayer::start,
+ ConnectionTypeBlocking);
+}
+
+void DebayerIsp::stop()
+{
+ if (!ispWorkerThread_.isRunning())
+ return;
+
+ debayer_->invokeMethod(&Debayer::stop,
+ ConnectionTypeBlocking);
+
+ RawIsp::stop();
+}
+
+void DebayerIsp::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output)
+{
+ ipa_->computeParams(frame);
+
+ debayer_->invokeMethod(&Debayer::process,
+ ConnectionTypeQueued, frame, input, output, debayerParams_);
+}
+
} /* namespace libcamera */
`simple' pipeline can basically pass raw buffers from V4L2 to the output. While the raw data is untouched, we may still want to: - Gather statistics from it. - Run IPA on the images (without modifying the images). - Provide metadata from the IPA. - Adjust exposure/gain. This patch implements all of that. Which introduces sort of processing to raw streams, making them similar to debayered streams. In order to not complicate `simple' pipeline more than necessary, SoftwareIsp class is split into a base class that serves just the raw streams, and a subclass with software ISP processing including debayering. Signed-off-by: Milan Zamazal <mzamazal@redhat.com> --- .../internal/software_isp/software_isp.h | 102 ++-- src/libcamera/pipeline/simple/simple.cpp | 63 +- src/libcamera/software_isp/software_isp.cpp | 540 ++++++++++++------ 3 files changed, 485 insertions(+), 220 deletions(-)