[v6,4/4] libcamera: software_isp: Add support for monochrome formats - GPU

28496 diff mbox series

Signed-off-by: Alain Cousinié <alain.cousinie@laposte.net>
---
 src/libcamera/software_isp/debayer_egl.cpp | 137 +++++++++++++++++++--
 src/libcamera/software_isp/debayer_egl.h   |   2 +
 2 files changed, 127 insertions(+), 12 deletions(-)

--
2.55.0

On 17/10/2026 12:10, alain.cousinie@laposte.net wrote:
> Signed-off-by: Alain Cousinié <alain.cousinie@laposte.net>
> ---
>   src/libcamera/software_isp/debayer_egl.cpp | 137 +++++++++++++++++++--
>   src/libcamera/software_isp/debayer_egl.h   |   2 +
>   2 files changed, 127 insertions(+), 12 deletions(-)
> 
> diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp
> index 300822f9d..00c680d78 100644
> --- a/src/libcamera/software_isp/debayer_egl.cpp
> +++ b/src/libcamera/software_isp/debayer_egl.cpp
> @@ -61,6 +61,21 @@ int DebayerEGL::getInputConfig(PixelFormat inputFormat, DebayerInputConfig &conf
>   						   formats::XBGR8888,
>   						   formats::ABGR8888 };
> 
> +	if (bayerFormat.order == BayerFormat::Order::MONO) {
> +		if (bayerFormat.packing == BayerFormat::Packing::None) {
> +			config.bpp = (bayerFormat.bitDepth + 7) & ~7;
> +			config.patternSize.width = 2;
> +			config.patternSize.height = 2;
> +		} else if (bayerFormat.packing == BayerFormat::Packing::CSI2) {
> +			config.bpp = bayerFormat.bitDepth;
> +			config.patternSize.width = (bayerFormat.bitDepth == 10) ? 4 : 2;
> +			config.patternSize.height = 2;
> +		}
> +
> +		config.outputFormats = outputFormats;
> +		return 0;
> +	}
> +
>   	if ((bayerFormat.bitDepth == 8 || bayerFormat.bitDepth == 10) &&
>   	    bayerFormat.packing == BayerFormat::Packing::None &&
>   	    isStandardBayerOrder(bayerFormat.order)) {
> @@ -166,6 +181,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
>   	shaderStridePixels_ = inputConfig_.stride;
> 
>   	switch (inputFormat) {
> +	case libcamera::formats::R8:
> +	case libcamera::formats::R10:
> +	case libcamera::formats::R12:
> +	case libcamera::formats::R16:
> +		firstRed_x_ = 0.0;
> +		firstRed_y_ = 0.0;
> +		break;
>   	case libcamera::formats::SBGGR8:
>   	case libcamera::formats::SBGGR10_CSI2P:
>   	case libcamera::formats::SBGGR12_CSI2P:
> @@ -197,6 +219,74 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
> 
>   	/* Shader selection */
>   	switch (inputFormat) {
> +	case libcamera::formats::R8:
> +		egl_.pushEnv(shaderEnv, "#define INV_MAX_VAL (1.0 / 255.0)");
> +		egl_.pushEnv(shaderEnv, "#define RAW8");
> +		fragmentShaderData = mono_unpacked_frag;
> +		vertexShaderData = mono_vert;
> +		glFormat_ = GL_LUMINANCE;
> +		bytesPerPixel_ = 1;
> +		break;
> +	case libcamera::formats::R10: {
> +		BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputFormat);
> +
> +		egl_.pushEnv(shaderEnv, "#define INV_MAX_VAL (1.0 / 1023.0)");
> +		if (bayerFormat.packing == BayerFormat::Packing::None) {
> +			egl_.pushEnv(shaderEnv, "#define RAW10");
> +			fragmentShaderData = mono_unpacked_frag;
> +			vertexShaderData = mono_vert;
> +			glFormat_ = GL_RG;
> +			bytesPerPixel_ = 2;
> +		} else {
> +			egl_.pushEnv(shaderEnv, "#define BLOCK_PIXELS 4.0");
> +			egl_.pushEnv(shaderEnv, "#define BLOCK_BYTES 5.0");
> +			egl_.pushEnv(shaderEnv, "#define LSB_SIZE 2.0");
> +			egl_.pushEnv(shaderEnv, "#define LSB_MASK_VAL 4.0");
> +			egl_.pushEnv(shaderEnv, (std::string("#define TEX_WIDTH ") + std::to_string(inputConfig_.stride) + ".0").c_str());
> +			egl_.pushEnv(shaderEnv, (std::string("#define TEX_HEIGHT ") + std::to_string(height_) + ".0").c_str());
> +
> +			fragmentShaderData = mono_1x_packed_frag;
> +			vertexShaderData = mono_vert;
> +			glFormat_ = GL_LUMINANCE;
> +			bytesPerPixel_ = 1;
> +			shaderStridePixels_ = inputConfig_.stride;
> +		}
> +		break;
> +	}
> +	case libcamera::formats::R12: {
> +		BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputFormat);
> +
> +		egl_.pushEnv(shaderEnv, "#define INV_MAX_VAL (1.0 / 4095.0)");
> +		if (bayerFormat.packing == BayerFormat::Packing::None) {
> +			egl_.pushEnv(shaderEnv, "#define RAW12");
> +			fragmentShaderData = mono_unpacked_frag;
> +			vertexShaderData = mono_vert;
> +			glFormat_ = GL_RG;
> +			bytesPerPixel_ = 2;
> +		} else {
> +			egl_.pushEnv(shaderEnv, "#define BLOCK_PIXELS 2.0");
> +			egl_.pushEnv(shaderEnv, "#define BLOCK_BYTES 3.0");
> +			egl_.pushEnv(shaderEnv, "#define LSB_SIZE 4.0");
> +			egl_.pushEnv(shaderEnv, "#define LSB_MASK_VAL 16.0");
> +			egl_.pushEnv(shaderEnv, (std::string("#define TEX_WIDTH ") + std::to_string(inputConfig_.stride) + ".0").c_str());
> +			egl_.pushEnv(shaderEnv, (std::string("#define TEX_HEIGHT ") + std::to_string(height_) + ".0").c_str());
> +
> +			fragmentShaderData = mono_1x_packed_frag;
> +			vertexShaderData = mono_vert;
> +			glFormat_ = GL_LUMINANCE;
> +			bytesPerPixel_ = 1;
> +			shaderStridePixels_ = inputConfig_.stride;
> +		}
> +		break;
> +	}
> +	case libcamera::formats::R16:
> +		egl_.pushEnv(shaderEnv, "#define INV_MAX_VAL (1.0 / 65535.0)");
> +		egl_.pushEnv(shaderEnv, "#define RAW16");
> +		fragmentShaderData = mono_unpacked_frag;
> +		vertexShaderData = mono_vert;
> +		glFormat_ = GL_RG;
> +		bytesPerPixel_ = 2;
> +		break;
>   	case libcamera::formats::SBGGR8:
>   	case libcamera::formats::SGBRG8:
>   	case libcamera::formats::SGRBG8:
> @@ -273,7 +363,7 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
> 
>   int DebayerEGL::configure(const StreamConfiguration &inputCfg,
>   			  const std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
> -			  [[maybe_unused]] bool ccmEnabled)
> +			  bool ccmEnabled)
>   {
>   	if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) {
>   		LOG(Debayer, Error)
> @@ -293,6 +383,15 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
>   		return -EINVAL;
>   	}
> 
> +	BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat);
> +	if (bayerFormat.order == BayerFormat::Order::MONO) {
> +		/* \todo Disable CCM via IPA; force false here to prevent monochrome coloration */
> +		ccmEnabled = false;
> +	} else {
> +		/* Force ccmEnabled to true as it is not yet populated. */
> +		ccmEnabled = true;
> +	}

It seems to me, superficially anyway that this is the wrong place to 
flip this logic.

Presumably we are still doing some CCM calculation in the IPA - I wonder 
is it not possible to detect the pixel format in the main softwareISP 
logic and send the ccmEnabled flag to the softisp and IPA layer ...

Yeah I think it should be

/**
  * \brief Configure the SoftwareIsp object according to the passed in 
parameters
  * \param[in] inputCfg The input configuration
  * \param[in] outputCfgs The output configurations
  * \param[in] configInfo The IPA configuration data, received from the 
pipeline handler
  * \param[out] ipaControls The IPA controls to update
  * \return 0 on success, a negative errno on failure
  */
int SoftwareIsp::configure(const StreamConfiguration &inputCfg,
                            const 
std::vector<std::reference_wrapper<const StreamConfiguration>> &outputCfgs,
                            const ipa::softisp::IPAConfigInfo &configInfo,
                            ControlInfoMap *ipaControls)
{
         ASSERT(ipa_ && debayer_);

         int ret = ipa_->configure(configInfo, ipaControls);
         if (ret < 0)
                 return ret;

         ret = debayer_->configure(inputCfg, outputCfgs, ccmEnabled_);
         if (ret < 0)

We should

- Extend IPA::configure() to take a ccmEnabled_ or otherwise make that
   aware of the state of the CCM
- Pass the variable from SoftwareIsp::configure() to both
   ipa_->configure() and debayer_->configure()

i.e. do the pixel format lookup in SoftwareIsp::configure() and pass the 
bool into both, instead of flipping the bool in debayer_->configure() only.

I'm sure there is work in the ipa_ that we can avoid doing when we know 
the CCM souldn't be applied.
> +
>   	inputConfig_.stride = inputCfg.stride;
>   	inputPixelFormat_ = inputCfg.pixelFormat;
>   	width_ = inputCfg.size.width;
> @@ -322,10 +421,16 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
>   	outputSize_ = outputCfg.size;
>   	nativeOutputSize_ = outSizeRange.max;
> 
> -	window_.x = ((inputCfg.size.width - outputCfg.size.width) / 2) &
> -		    ~(inputConfig_.patternSize.width - 1);
> -	window_.y = ((inputCfg.size.height - outputCfg.size.height) / 2) &
> -		    ~(inputConfig_.patternSize.height - 1);
> +	/* Setup the window geometry according to the format type */
> +	if (bayerFormat.order == BayerFormat::Order::MONO) {
> +		window_.x = 0;
> +		window_.y = 0;
> +	} else {
> +		window_.x = ((inputCfg.size.width - outputCfg.size.width) / 2) &
> +			    ~(inputConfig_.patternSize.width - 1);
> +		window_.y = ((inputCfg.size.height - outputCfg.size.height) / 2) &
> +			    ~(inputConfig_.patternSize.height - 1);
> +	}
>   	window_.width = outputCfg.size.width;
>   	window_.height = outputCfg.size.height;
> 
> @@ -338,6 +443,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
>   	inputBufferCount_ = inputCfg.bufferCount;
>   	outputBufferCount_ = outputCfg.bufferCount;
> 
> +	ccmEnabled_ = ccmEnabled;
> +
>   	return 0;
>   }
> 
> @@ -465,9 +572,11 @@ void DebayerEGL::setShaderVariableValues(eGLImage &eglImageIn, const DebayerPara
>   			    << " textureUniformProjMatrix_ " << textureUniformProjMatrix_;
> 
>   	/* Pre-transpose matrix for GLES 2.0. */
> -	Matrix<float, 3, 3> ccm = params.combinedMatrix.transpose();
> -	glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm.data().data());
> -	LOG(Debayer, Debug) << " ccmUniformDataIn_ " << ccmUniformDataIn_ << " data " << params.combinedMatrix;
> +	if (ccmEnabled_) {
> +		Matrix<float, 3, 3> ccm = params.combinedMatrix.transpose();
> +		glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm.data().data());
> +		LOG(Debayer, Debug) << " ccmUniformDataIn_ " << ccmUniformDataIn_ << " data " << params.combinedMatrix;
> +	}
> 
>   	/*
>   	 * 0 = Red, 1 = Green, 2 = Blue
> @@ -692,11 +801,12 @@ void DebayerEGL::stop()
>   SizeRange DebayerEGL::sizes(PixelFormat inputFormat, const Size &inputSize)
>   {
>   	Size patternSize = this->patternSize(inputFormat);
> -	unsigned int borderHeight = patternSize.height;
> -
>   	if (patternSize.isNull())
>   		return {};
> 
> +	/* FIX: borderHeight is now safely initialized after the isNull() check */
> +	unsigned int borderHeight = patternSize.height;
> +
>   	/* No need for top/bottom border with a pattern height of 2 */
>   	if (patternSize.height == 2)
>   		borderHeight = 0;
> @@ -712,9 +822,12 @@ SizeRange DebayerEGL::sizes(PixelFormat inputFormat, const Size &inputSize)
>   		return {};
>   	}
> 
> +	BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputFormat);
> +	bool localIsMono = (bayerFormat.order == BayerFormat::Order::MONO);
> +
>   	return SizeRange(Size(patternSize.width, patternSize.height),
> -			 Size((inputSize.width - 2 * patternSize.width) & ~(patternSize.width - 1),
> -			      (inputSize.height - 2 * borderHeight) & ~(patternSize.height - 1)),
> +			 Size((inputSize.width - (localIsMono ? 0 : 2 * patternSize.width)) & ~(patternSize.width - 1),
> +			      (inputSize.height - (localIsMono ? 0 : 2 * borderHeight)) & ~(patternSize.height - 1)),
>   			 patternSize.width, patternSize.height);
>   }
> 
> diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h
> index 30e51a477..197a2b6f1 100644
> --- a/src/libcamera/software_isp/debayer_egl.h
> +++ b/src/libcamera/software_isp/debayer_egl.h
> @@ -119,6 +119,8 @@ private:
>   	GLint glFormat_;
>   	unsigned int bytesPerPixel_;
>   	uint32_t shaderStridePixels_;
> +
> +	bool ccmEnabled_;
>   };
> 
>   } /* namespace libcamera */
> --
> 2.55.0

Hi Bryan,

Thanks for your review and feedback on v6.

Regarding identity.vert vs mono.vert:
I prefer to keep mono.vert for three key reasons:
1. Speed: Moving the coordinate calculation (textureOut * tex_size) to the fragment shader means executing it millions of times per frame. In mono.vert, it runs only 4 times per frame (per vertex), saving significant GPU cycles on low-end hardware.
2. Precision: Vertex shaders guarantee highp precision across all OpenGL ES 2.0 implementations. This immunizes the bit-shifting math against float interpolation rounding errors if a legacy GPU falls back to mediump in the fragment shader.
3. Extensibility: Keeping a dedicated vertex shader provides a cleaner and more flexible foundation for future architectural evolutions planned for the monochrome pipeline.

Regarding the CCM/LSC handling:
Following the architectural discussions about the mismatch between init() and configure() lifecycles, I will centralize the temporary monochrome override directly within SoftwareIsp::configure() in software_isp.cpp for v7:

```cpp
int SoftwareIsp::configure(const StreamConfiguration &inputCfg, ...)
{
	ASSERT(ipa_ && debayer_);

	int ret = ipa_->configure(configInfo, ipaControls);
	if (ret < 0)
		return ret;

	BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat);
	if (bayerFormat.order == BayerFormat::Order::MONO) {
		/* \todo Move to IPA uncalibrated-mono.yaml when lifecycle allows */
		ccmEnabled_ = false;
		lscEnabled_ = false;
	}

	ret = debayer_->configure(inputCfg, outputCfgs, ccmEnabled_, lscEnabled_);
	...
}
```

This will completely clean up both debayer_cpu and debayer_egl backends, keeping them generic and free of format-specific overrides.

Additionally, looking at Jai's latest formatting and refactoring layout, I am considering introducing the R14 (unpacked and packed) formats. I might add them directly as a `\todo` comment in the formats tables to layout the groundwork for a future iteration.

What do you think?

Best regards,
Alain

On 24/10/2026 07:02, Alain Cousinie wrote:
> Hi Bryan,
> 
> Thanks for your review and feedback on v6.
> 
> Regarding identity.vert vs mono.vert:
> I prefer to keep mono.vert for three key reasons:
> 1. Speed: Moving the coordinate calculation (textureOut * tex_size) to the fragment shader means executing it millions of times per frame. In mono.vert, it runs only 4 times per frame (per vertex), saving significant GPU cycles on low-end hardware.

Fair point.

> 2. Precision: Vertex shaders guarantee highp precision across all OpenGL ES 2.0 implementations. This immunizes the bit-shifting math against float interpolation rounding errors if a legacy GPU falls back to mediump in the fragment shader.

Yes.

> 3. Extensibility: Keeping a dedicated vertex shader provides a cleaner and more flexible foundation for future architectural evolutions planned for the monochrome pipeline.
> Regarding the CCM/LSC handling:
> Following the architectural discussions about the mismatch between init() and configure() lifecycles, I will centralize the temporary monochrome override directly within SoftwareIsp::configure() in software_isp.cpp for v7:
> 
> ```cpp
> int SoftwareIsp::configure(const StreamConfiguration &inputCfg, ...)
> {
> 	ASSERT(ipa_ && debayer_);
> 
> 	int ret = ipa_->configure(configInfo, ipaControls);
> 	if (ret < 0)
> 		return ret;
> 
> 	BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat);
> 	if (bayerFormat.order == BayerFormat::Order::MONO) {
> 		/* \todo Move to IPA uncalibrated-mono.yaml when lifecycle allows */
> 		ccmEnabled_ = false;
> 		lscEnabled_ = false;
> 	}

Yes.

> 
> 	ret = debayer_->configure(inputCfg, outputCfgs, ccmEnabled_, lscEnabled_);
> 	...
> }
> ```
> 
> This will completely clean up both debayer_cpu and debayer_egl backends, keeping them generic and free of format-specific overrides.
> 
> Additionally, looking at Jai's latest formatting and refactoring layout, I am considering introducing the R14 (unpacked and packed) formats. I might add them directly as a `\todo` comment in the formats tables to layout the groundwork for a future iteration.
> 
> What do you think?

If you can test it on actual hardware go ahead.

> Best regards,
> Alain

Hi Alain,

Alain Cousinie <alain.cousinie@laposte.net> writes:

> Regarding identity.vert vs mono.vert:
> I prefer to keep mono.vert for three key reasons:
> 1. Speed: Moving the coordinate calculation (textureOut * tex_size) to the fragment shader means executing it millions of times per frame. In mono.vert, it runs only 4 times per frame (per vertex), saving significant GPU cycles on low-end hardware.
> 2. Precision: Vertex shaders guarantee highp precision across all OpenGL ES 2.0 implementations. This immunizes the bit-shifting math against float interpolation rounding errors if a legacy GPU falls back to mediump in the fragment shader.

But 1. + 2. applies also to the non-mono case, right?  Then the changes
should be applied to identity.vert, shouldn't they?  And then the
shaders would be identical.

> 3. Extensibility: Keeping a dedicated vertex shader provides a cleaner and more flexible foundation for future architectural evolutions planned for the monochrome pipeline.

Once there are reasons why the shaders shouldn't be identical, let's
split them.  Not the case now AFAIUI.