[12/30] libcamera: shaders: Split packed and unpacked demosiac up

26945 diff mbox series

Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
---
 .../bayer_1x_packed_to_blc_glr16f.frag        |  97 ++++++++++
 .../shaders/bayer_glr16_to_rgba.frag          | 183 ++++++++++++++++++
 .../shaders/bayer_unpacked_to_blc_glr16f.frag |  46 +++++
 src/libcamera/shaders/meson.build             |   3 +
 4 files changed, 329 insertions(+)
 create mode 100644 src/libcamera/shaders/bayer_1x_packed_to_blc_glr16f.frag
 create mode 100644 src/libcamera/shaders/bayer_glr16_to_rgba.frag
 create mode 100644 src/libcamera/shaders/bayer_unpacked_to_blc_glr16f.frag

s/demosiac/demosaic/ in any past or future patches. :-)

Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes:

> Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
> ---
>  .../bayer_1x_packed_to_blc_glr16f.frag        |  97 ++++++++++
>  .../shaders/bayer_glr16_to_rgba.frag          | 183 ++++++++++++++++++
>  .../shaders/bayer_unpacked_to_blc_glr16f.frag |  46 +++++
>  src/libcamera/shaders/meson.build             |   3 +
>  4 files changed, 329 insertions(+)
>  create mode 100644 src/libcamera/shaders/bayer_1x_packed_to_blc_glr16f.frag
>  create mode 100644 src/libcamera/shaders/bayer_glr16_to_rgba.frag
>  create mode 100644 src/libcamera/shaders/bayer_unpacked_to_blc_glr16f.frag
>
> diff --git a/src/libcamera/shaders/bayer_1x_packed_to_blc_glr16f.frag b/src/libcamera/shaders/bayer_1x_packed_to_blc_glr16f.frag
> new file mode 100644
> index 000000000..d1234007e
> --- /dev/null
> +++ b/src/libcamera/shaders/bayer_1x_packed_to_blc_glr16f.frag
> @@ -0,0 +1,97 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/
> + *
> + * Copyright (c) 2008, Morgan McGuire. All rights reserved.
> + *
> + * Modified by Linaro Ltd for 10/12-bit packed raw Bayer format.
> + * Copyright (C) 2020, Linaro
> + *
> + * precursor_packed.frag - Precursor shader to decode MIPI packed raw Bayer

s/precursor_packed.frag - //

> + * data and output normalised single-channel float to an R16F FBO.
> + * Pairs with identity.vert.
> + */
> +
> +#ifdef GL_ES
> +precision highp float;
> +#endif
> +
> +/*
> + * These constants are used to select the bytes containing the HS part of
> + * the pixel value:
> + * BPP - bytes per pixel,
> + * THRESHOLD_L = fract(BPP) * 0.5 + 0.02
> + * THRESHOLD_H = 1.0 - fract(BPP) * 1.5 + 0.02
> + * Let X is the x coordinate in the texture measured in bytes (so that the
> + * range is from 0 to (stride_-1)) aligned on the nearest pixel.
> + * E.g. for RAW10P:
> + * -------------+-------------------+-------------------+--
> + *  pixel No    |  0   1    2   3   |  4   5    6   7   | ...
> + * -------------+-------------------+-------------------+--
> + *  byte offset | 0   1   2   3   4 | 5   6   7   8   9 | ...
> + * -------------+-------------------+-------------------+--
> + *      X       | 0.0 1.25 2.5 3.75 | 5.0 6.25 7.5 8.75 | ...
> + * -------------+-------------------+-------------------+--
> + * If fract(X) < THRESHOLD_L then the previous byte contains the LS
> + * bits of the pixel values and needs to be skipped.
> + * If fract(X) > THRESHOLD_H then the next byte contains the LS bits
> + * of the pixel values and needs to be skipped.
> + */
> +#if defined(RAW10P)
> +#define BPP		1.25
> +#define THRESHOLD_L	0.14
> +#define THRESHOLD_H	0.64
> +#elif defined(RAW12P)
> +#define BPP		1.5
> +#define THRESHOLD_L	0.27
> +#define THRESHOLD_H	0.27
> +#else
> +#error Invalid raw format
> +#endif
> +
> +varying vec2 textureOut;
> +
> +/* the texture size in pixels */
> +uniform vec2 tex_size;
> +uniform vec2 tex_step;
> +
> +uniform vec3 blacklevel;
> +
> +uniform sampler2D tex_y;
> +
> +void main(void)
> +{
> +	/*
> +	 * center_bytes holds the coordinates of the MS byte of the pixel
> +	 * being sampled on the [0, stride-1/height-1] range.
> +	 * center_pixel holds the coordinates of the pixel being sampled
> +	 * on the [0, width/height-1] range.
> +	 */
> +	vec2 center_bytes;
> +	vec2 center_pixel;
> +
> +	/*
> +	 * The coordinates passed to the shader in textureOut may point
> +	 * to a place in between the pixels if the texture format doesn't
> +	 * match the image format. In particular, MIPI packed raw Bayer
> +	 * formats don't have a matching texture format.
> +	 * In this case align the coordinates to the left nearest pixel
> +	 * by hand.
> +	 */
> +	center_pixel = floor(textureOut * tex_size);
> +	center_bytes.y = center_pixel.y;
> +
> +	/*
> +	 * Add a small number (a few mantissa's LSBs) to avoid float
> +	 * representation issues.
> +	 */
> +	center_bytes.x = BPP * center_pixel.x + 0.02;
> +	center_bytes.x = floor(center_bytes.x);
> +	center_bytes *= tex_step;
> +
> +	float C = texture2D(tex_y, center_bytes).r;
> +
> +	C = C - blacklevel[0];

The black level is subtracted only here?  Not in the unpacked case?
Shouldn't it be subtracted in the common shader?

> +
> +	gl_FragColor = vec4(C, 0.0, 0.0, 0.0);

Alpha should be 1.0, at least for consistency.

> +}
> diff --git a/src/libcamera/shaders/bayer_glr16_to_rgba.frag b/src/libcamera/shaders/bayer_glr16_to_rgba.frag
> new file mode 100644
> index 000000000..f3883a82b
> --- /dev/null
> +++ b/src/libcamera/shaders/bayer_glr16_to_rgba.frag
> @@ -0,0 +1,183 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> +From http://jgt.akpeters.com/papers/McGuire08/
> +
> +Efficient, High-Quality Bayer Demosaic Filtering on GPUs
> +
> +Morgan McGuire
> +
> +This paper appears in issue Volume 13, Number 4.
> +---------------------------------------------------------
> +Copyright (c) 2008, Morgan McGuire. All rights reserved.
> +
> +Modified by Linaro Ltd to integrate it into libcamera.
> +Copyright (C) 2021-2026, Linaro

Pairs with bayer_unpacked.vert?  (Which should be renamed then.)

> +*/
> +
> +//Pixel Shader
> +#ifdef GL_ES
> +precision highp float;
> +#endif
> +
> +/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/
> +uniform sampler2D	tex_y;
> +varying vec4		center;
> +varying vec4		yCoord;
> +varying vec4		xCoord;
> +uniform mat3		ccm;
> +uniform float		gamma;
> +uniform float		contrastExp;
> +
> +float apply_contrast(float value)
> +{
> +	// Apply simple S-curve
> +	if (value < 0.5)
> +		return 0.5 * pow(value / 0.5, contrastExp);
> +	else
> +		return 1.0 - 0.5 * pow((1.0 - value) / 0.5, contrastExp);
> +}
> +
> +void main(void) {
> +	vec3 rgb;
> +
> +	/* Sample from R16F input texture */
> +	#define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
> +
> +	float C = fetch(center.x, center.y); // ( 0, 0)
> +	const vec4 kC = vec4( 4.0,  6.0,  5.0,  5.0) / 8.0;
> +
> +	// Determine which of four types of pixels we are on.
> +	vec2 alternate = mod(floor(center.zw), 2.0);
> +
> +	vec4 Dvec = vec4(
> +		fetch(xCoord[1], yCoord[1]),  // (-1,-1)
> +		fetch(xCoord[1], yCoord[2]),  // (-1, 1)
> +		fetch(xCoord[2], yCoord[1]),  // ( 1,-1)
> +		fetch(xCoord[2], yCoord[2])); // ( 1, 1)
> +
> +	vec4 PATTERN = (kC.xyz * C).xyzz;
> +
> +	// Can also be a dot product with (1,1,1,1) on hardware where that is
> +	// specially optimized.
> +	// Equivalent to: D = Dvec[0] + Dvec[1] + Dvec[2] + Dvec[3];
> +	Dvec.xy += Dvec.zw;
> +	Dvec.x  += Dvec.y;
> +
> +	vec4 value = vec4(
> +		fetch(center.x, yCoord[0]),   // ( 0,-2)
> +		fetch(center.x, yCoord[1]),   // ( 0,-1)
> +		fetch(xCoord[0], center.y),   // (-2, 0)
> +		fetch(xCoord[1], center.y));  // (-1, 0)
> +
> +	vec4 temp = vec4(
> +		fetch(center.x, yCoord[3]),   // ( 0, 2)
> +		fetch(center.x, yCoord[2]),   // ( 0, 1)
> +		fetch(xCoord[3], center.y),   // ( 2, 0)
> +		fetch(xCoord[2], center.y));  // ( 1, 0)
> +
> +	// Even the simplest compilers should be able to constant-fold these to
> +	// avoid the division.
> +	// Note that on scalar processors these constants force computation of some
> +	// identical products twice.
> +	const vec4 kA = vec4(-1.0, -1.5,  0.5, -1.0) / 8.0;
> +	const vec4 kB = vec4( 2.0,  0.0,  0.0,  4.0) / 8.0;
> +	const vec4 kD = vec4( 0.0,  2.0, -1.0, -1.0) / 8.0;
> +
> +	// Conserve constant registers and take advantage of free swizzle on load
> +	#define kE (kA.xywz)
> +	#define kF (kB.xywz)
> +
> +	value += temp;
> +
> +	// There are five filter patterns (identity, cross, checker,
> +	// theta, phi).  Precompute the terms from all of them and then
> +	// use swizzles to assign to color channels.
> +	//
> +	// Channel   Matches
> +	//   x       cross   (e.g., EE G)
> +	//   y       checker (e.g., EE B)
> +	//   z       theta   (e.g., EO R)
> +	//   w       phi     (e.g., EO R)
> +	#define A (value[0])
> +	#define B (value[1])
> +	#define D (Dvec.x)
> +	#define E (value[2])
> +	#define F (value[3])
> +
> +	// Avoid zero elements. On a scalar processor this saves two MADDs
> +	// and it has no effect on a vector processor.
> +	PATTERN.yzw += (kD.yz * D).xyy;
> +
> +	PATTERN += (kA.xyz * A).xyzx + (kE.xyw * E).xyxz;
> +	PATTERN.xw  += kB.xw * B;
> +	PATTERN.xz  += kF.xz * F;
> +
> +	rgb =  (alternate.y == 0.0) ?
> +	((alternate.x == 0.0) ?
> +		vec3(C, PATTERN.xy) :
> +		vec3(PATTERN.z, C, PATTERN.w)) :
> +	((alternate.x == 0.0) ?
> +		vec3(PATTERN.w, C, PATTERN.z) :
> +		vec3(PATTERN.yx, C));
> +
> +	/*
> +	*   CCM is a 3x3 in the format
> +	*
> +	*   +--------------+----------------+---------------+
> +	*   | RedRedGain   | RedGreenGain   | RedBlueGain   |
> +	*   +--------------+----------------+---------------+
> +	*   | GreenRedGain | GreenGreenGain | GreenBlueGain |
> +	*   +--------------+----------------+---------------+
> +	*   | BlueRedGain  |  BlueGreenGain | BlueBlueGain  |
> +	*   +--------------+----------------+---------------+
> +	*
> +	*   Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin
> +	*   Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin
> +	*   Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin
> +	*
> +	*   We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm);
> +	*
> +	*   CPU
> +	*   float ccm [] = {
> +	*             RedRedGain,   RedGreenGain,   RedBlueGain,
> +	*             GreenRedGain, GreenGreenGain, GreenBlueGain,
> +	*             BlueRedGain,  BlueGreenGain,  BlueBlueGain,
> +	*   };
> +	*
> +	*   GPU
> +	*   ccm = {
> +	*             RedRedGain,   GreenRedGain,   BlueRedGain,
> +	*             RedGreenGain, GreenGreenGain, BlueGreenGain,
> +	*             RedBlueGain,  GreenBlueGain,  BlueBlueGain,
> +	*   }
> +	*
> +	*   However the indexing for the mat data-type is column major hence
> +	*   ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain
> +	*
> +	*/
> +	float rin, gin, bin;
> +	rin = rgb.r;
> +	gin = rgb.g;
> +	bin = rgb.b;
> +
> +	rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]);
> +	rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]);
> +	rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]);
> +
> +	/*
> +	 * Contrast
> +	 */
> +	rgb = clamp(rgb, 0.0, 1.0);
> +	rgb.r = apply_contrast(rgb.r);
> +	rgb.g = apply_contrast(rgb.g);
> +	rgb.b = apply_contrast(rgb.b);
> +
> +	/* Apply gamma after colour correction */
> +	rgb = pow(rgb, vec3(gamma));
> +
> +	#if defined (SWAP_BLUE)
> +		gl_FragColor = vec4(rgb.bgr, 1.0);
> +	#else
> +		gl_FragColor = vec4(rgb, 1.0);
> +	#endif
> +}

Since this copies, rather than moves, code, beware of possible unnoticed
changes to the shaders in the meantime.

> diff --git a/src/libcamera/shaders/bayer_unpacked_to_blc_glr16f.frag b/src/libcamera/shaders/bayer_unpacked_to_blc_glr16f.frag
> new file mode 100644
> index 000000000..00ed47ce8
> --- /dev/null
> +++ b/src/libcamera/shaders/bayer_unpacked_to_blc_glr16f.frag
> @@ -0,0 +1,46 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/
> + *
> + * Copyright (c) 2008, Morgan McGuire. All rights reserved.

I don't think the lines above apply here.

> + *
> + * Modified by Linaro Ltd to integrate it into libcamera.
> + * Copyright (C) 2021, Linaro
> + *
> + * precursor_unpacked.frag - Precursor shader to decode unpacked raw Bayer

s/precursor_unpacked.frag - //

> + * data (R8, RG8 with split 10/12-bit values) and output normalised
> + * single-channel float to an R16F FBO.
> + * Pairs with identity.vert.
> + */
> +
> +#ifdef GL_ES
> +precision highp float;
> +#endif
> +
> +varying vec2 textureOut;
> +
> +uniform sampler2D tex_y;
> +
> +void main(void)
> +{
> +	/*
> +	 * Reconstruct the pixel value from the texture format.
> +	 *
> +	 * For RAW10P the 10-bit value is split across two 8-bit channels:
> +	 *   value = R / 4.0 + G * 64.0
> +	 * For RAW12P the 12-bit value is split across two 8-bit channels:
> +	 *   value = R / 16.0 + G * 16.0
> +	 * Otherwise the value is a plain single-channel sample.
> +	 */
> +#if defined(RAW10P)
> +	vec4 p = texture2D(tex_y, textureOut);
> +	float C = p.r / 4.0 + p.g * 64.0;
> +#elif defined(RAW12P)
> +	vec4 p = texture2D(tex_y, textureOut);
> +	float C = p.r / 16.0 + p.g * 16.0;
> +#else
> +	float C = texture2D(tex_y, textureOut).r;
> +#endif
> +
> +	gl_FragColor = vec4(C, 0.0, 0.0, 1.0);
> +}
> diff --git a/src/libcamera/shaders/meson.build b/src/libcamera/shaders/meson.build
> index dd441a577..0e2b765d7 100644
> --- a/src/libcamera/shaders/meson.build
> +++ b/src/libcamera/shaders/meson.build
> @@ -3,6 +3,9 @@
>  # List of shader files to convert to header hex
>  # for the purposes of inclusion in OpenGL debayering
>  shader_files = files([
> +    'bayer_1x_packed_to_blc_glr16f.frag',
> +    'bayer_unpacked_to_blc_glr16f.frag',
> +    'bayer_glr16_to_rgba.frag',
>      'bayer_1x_packed.frag',
>      'bayer_unpacked.frag',
>      'bayer_unpacked.vert',