[libcamera-devel,7/7] qcam: viewfinder_gl: Add shader to render packed YUV formats

9643 diff mbox series

The shader supports all 4 packed 8-bit YUV variants.

Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
---
 src/qcam/assets/shader/YUV_packed.frag | 82 ++++++++++++++++++++++++++
 src/qcam/assets/shader/shaders.qrc     |  1 +
 src/qcam/viewfinder_gl.cpp             | 56 ++++++++++++++++++
 src/qcam/viewfinder_gl.h               |  1 +
 4 files changed, 140 insertions(+)
 create mode 100644 src/qcam/assets/shader/YUV_packed.frag

Hi Laurent,

Thanks for your work.

On 2020-09-16 17:52:54 +0300, Laurent Pinchart wrote:
> The shader supports all 4 packed 8-bit YUV variants.

OpenGL is not my strong suite so I can't really give it a proper review.  
From what I do understand this change looks good but I'm unable to give 
a R-b in good concious with my limited understanding. I do however think 
the change is neatly modularized and properly tested with good results 
so I think merging it adds value without any risk for regression or 
later problems.

Acked-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>

Please see bellow for a small typo tho.

> 
> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> ---
>  src/qcam/assets/shader/YUV_packed.frag | 82 ++++++++++++++++++++++++++
>  src/qcam/assets/shader/shaders.qrc     |  1 +
>  src/qcam/viewfinder_gl.cpp             | 56 ++++++++++++++++++
>  src/qcam/viewfinder_gl.h               |  1 +
>  4 files changed, 140 insertions(+)
>  create mode 100644 src/qcam/assets/shader/YUV_packed.frag
> 
> diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag
> new file mode 100644
> index 000000000000..224dfafe383e
> --- /dev/null
> +++ b/src/qcam/assets/shader/YUV_packed.frag
> @@ -0,0 +1,82 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> + *
> + * YUV_packed.frag - Fragment shader code for YUYV packed formats
> + */
> +
> +#ifdef GL_ES
> +precision mediump float;
> +#endif
> +
> +varying vec2 textureOut;
> +
> +uniform sampler2D tex_y;
> +uniform float tex_stepx;
> +
> +void main(void)
> +{
> +	mat3 yuv2rgb_bt601_mat = mat3(
> +		vec3(1.164,  1.164, 1.164),
> +		vec3(0.000, -0.392, 2.017),
> +		vec3(1.596, -0.813, 0.000)
> +	);
> +	vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500);
> +
> +	/*
> +	 * The sampler won't interpolate the texture correctly along the X axis,
> +	 * as each RGBA pixel effectively stores two pixels. We thus need to
> +	 * interpolate manually.
> +	 *
> +	 * In integer texture coordinates, the Y values are layed out in the
> +	 * texture memory as follows:
> +	 *
> +	 * ...| Y  U  Y  V | Y  U  Y  V | Y  U  Y  V |...
> +	 * ...| R  G  B  A | R  G  B  A | R  G  B  A |...
> +	 *      ^     ^      ^     ^      ^     ^
> +	 *      |     |      |     |      |     |
> +	 *     n-1  n-0.5    n   n+0.5   n+1  n+1.5
> +	 *
> +	 * For a texture location x in the interval [n, n+1[, sample the left

s/[n, n+1[/[n, n+1]/

> +	 * and right pixels at n and n+1, and interpolate them with
> +	 *
> +	 * left.r * (1 - a) + left.b * a	if fract(x) < 0.5
> +	 * left.b * (1 - a) + right.r * a	if fract(x) >= 0.5
> +	 *
> +	 * with a = fract(x * 2) which can also be written
> +	 *
> +	 * a = fract(x) * 2			if fract(x) < 0.5
> +	 * a = fract(x) * 2 - 1			if fract(x) >= 0.5
> +	 */
> +	vec2 pos = textureOut;
> +	float f_x = fract(pos.x / tex_stepx);
> +
> +	vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y));
> +	vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y));
> +
> +#if defined(YUV_PATTERN_UYVY)
> +	float y_left = mix(left.g, left.a, f_x * 2.0);
> +	float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> +	vec2 uv = mix(left.rb, right.rb, f_x);
> +#elif defined(YUV_PATTERN_VYUY)
> +	float y_left = mix(left.g, left.a, f_x * 2.0);
> +	float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> +	vec2 uv = mix(left.br, right.br, f_x);
> +#elif defined(YUV_PATTERN_YUYV)
> +	float y_left = mix(left.r, left.b, f_x * 2.0);
> +	float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> +	vec2 uv = mix(left.ga, right.ga, f_x);
> +#elif defined(YUV_PATTERN_YVYU)
> +	float y_left = mix(left.r, left.b, f_x * 2.0);
> +	float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> +	vec2 uv = mix(left.ag, right.ag, f_x);
> +#else
> +#error Invalid pattern
> +#endif
> +
> +	float y = mix(y_left, y_right, step(0.5, f_x));
> +
> +	vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset);
> +
> +	gl_FragColor = vec4(rgb, 1.0);
> +}
> diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> index 7010d8433c9b..857ed9fd5cde 100644
> --- a/src/qcam/assets/shader/shaders.qrc
> +++ b/src/qcam/assets/shader/shaders.qrc
> @@ -4,5 +4,6 @@
>  	<file>YUV.vert</file>
>  	<file>YUV_2_planes.frag</file>
>  	<file>YUV_3_planes.frag</file>
> +	<file>YUV_packed.frag</file>
>  </qresource>
>  </RCC>
> diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> index b8a4827267c3..0b5c942658cd 100644
> --- a/src/qcam/viewfinder_gl.cpp
> +++ b/src/qcam/viewfinder_gl.cpp
> @@ -14,12 +14,19 @@
>  #include <libcamera/formats.h>
>  
>  static const QList<libcamera::PixelFormat> supportedFormats{
> +	/* Packed (single plane) */
> +	libcamera::formats::UYVY,
> +	libcamera::formats::VYUY,
> +	libcamera::formats::YUYV,
> +	libcamera::formats::YVYU,
> +	/* Semi planar (two planes) */
>  	libcamera::formats::NV12,
>  	libcamera::formats::NV21,
>  	libcamera::formats::NV16,
>  	libcamera::formats::NV61,
>  	libcamera::formats::NV24,
>  	libcamera::formats::NV42,
> +	/* Fully planar (three planes) */
>  	libcamera::formats::YUV420,
>  	libcamera::formats::YVU420,
>  };
> @@ -149,6 +156,22 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
>  		vertSubSample_ = 2;
>  		fragmentShaderFile_ = ":YUV_3_planes.frag";
>  		break;
> +	case libcamera::formats::UYVY:
> +		fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY");
> +		fragmentShaderFile_ = ":YUV_packed.frag";
> +		break;
> +	case libcamera::formats::VYUY:
> +		fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY");
> +		fragmentShaderFile_ = ":YUV_packed.frag";
> +		break;
> +	case libcamera::formats::YUYV:
> +		fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV");
> +		fragmentShaderFile_ = ":YUV_packed.frag";
> +		break;
> +	case libcamera::formats::YVYU:
> +		fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU");
> +		fragmentShaderFile_ = ":YUV_packed.frag";
> +		break;
>  	default:
>  		ret = false;
>  		qWarning() << "[ViewFinderGL]:"
> @@ -235,6 +258,7 @@ bool ViewFinderGL::createFragmentShader()
>  	textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
>  	textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
>  	textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
> +	textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx");
>  
>  	if (!textureY_.isCreated())
>  		textureY_.create();
> @@ -431,6 +455,38 @@ void ViewFinderGL::doRender()
>  		shaderProgram_.setUniformValue(textureUniformU_, 1);
>  		break;
>  
> +	case libcamera::formats::UYVY:
> +	case libcamera::formats::VYUY:
> +	case libcamera::formats::YUYV:
> +	case libcamera::formats::YVYU:
> +		/*
> +		 * Packed YUV formats are stored in a RGBA texture to match the
> +		 * OpenGL texel size with the 4 bytes repeating pattern in YUV.
> +		 * The texture width is thus half of the image with.
> +		 */
> +		glActiveTexture(GL_TEXTURE0);
> +		configureTexture(textureY_);
> +		glTexImage2D(GL_TEXTURE_2D,
> +			     0,
> +			     GL_RGBA,
> +			     size_.width() / 2,
> +			     size_.height(),
> +			     0,
> +			     GL_RGBA,
> +			     GL_UNSIGNED_BYTE,
> +			     yuvData_);
> +		shaderProgram_.setUniformValue(textureUniformY_, 0);
> +
> +		/*
> +		 * The shader needs the step between two texture pixels in the
> +		 * horizontal direction, expressed in texture coordinate units
> +		 * ([0, 1]). There are exactly width - 1 steps between the
> +		 * leftmost and rightmost texels.
> +		 */
> +		shaderProgram_.setUniformValue(textureUniformStepX_,
> +					       1.0f / (size_.width() / 2 - 1));
> +		break;
> +
>  	default:
>  		break;
>  	};
> diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
> index 53424dc10bc5..ad1e195e45c7 100644
> --- a/src/qcam/viewfinder_gl.h
> +++ b/src/qcam/viewfinder_gl.h
> @@ -79,6 +79,7 @@ private:
>  	GLuint textureUniformU_;
>  	GLuint textureUniformV_;
>  	GLuint textureUniformY_;
> +	GLuint textureUniformStepX_;
>  	QOpenGLTexture textureU_;
>  	QOpenGLTexture textureV_;
>  	QOpenGLTexture textureY_;
> -- 
> Regards,
> 
> Laurent Pinchart
> 
> _______________________________________________
> libcamera-devel mailing list
> libcamera-devel@lists.libcamera.org
> https://lists.libcamera.org/listinfo/libcamera-devel

Hi Niklas,

On Wed, Sep 16, 2020 at 05:55:20PM +0200, Niklas Söderlund wrote:
> Hi Laurent,
> 
> Thanks for your work.
> 
> On 2020-09-16 17:52:54 +0300, Laurent Pinchart wrote:
> > The shader supports all 4 packed 8-bit YUV variants.
> 
> OpenGL is not my strong suite so I can't really give it a proper review.  
> From what I do understand this change looks good but I'm unable to give 
> a R-b in good concious with my limited understanding. I do however think 
> the change is neatly modularized and properly tested with good results 
> so I think merging it adds value without any risk for regression or 
> later problems.
> 
> Acked-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
> 
> Please see bellow for a small typo tho.
> 
> > Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> > ---
> >  src/qcam/assets/shader/YUV_packed.frag | 82 ++++++++++++++++++++++++++
> >  src/qcam/assets/shader/shaders.qrc     |  1 +
> >  src/qcam/viewfinder_gl.cpp             | 56 ++++++++++++++++++
> >  src/qcam/viewfinder_gl.h               |  1 +
> >  4 files changed, 140 insertions(+)
> >  create mode 100644 src/qcam/assets/shader/YUV_packed.frag
> > 
> > diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag
> > new file mode 100644
> > index 000000000000..224dfafe383e
> > --- /dev/null
> > +++ b/src/qcam/assets/shader/YUV_packed.frag
> > @@ -0,0 +1,82 @@
> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> > +/*
> > + * Copyright (C) 2020, Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> > + *
> > + * YUV_packed.frag - Fragment shader code for YUYV packed formats
> > + */
> > +
> > +#ifdef GL_ES
> > +precision mediump float;
> > +#endif
> > +
> > +varying vec2 textureOut;
> > +
> > +uniform sampler2D tex_y;
> > +uniform float tex_stepx;
> > +
> > +void main(void)
> > +{
> > +	mat3 yuv2rgb_bt601_mat = mat3(
> > +		vec3(1.164,  1.164, 1.164),
> > +		vec3(0.000, -0.392, 2.017),
> > +		vec3(1.596, -0.813, 0.000)
> > +	);
> > +	vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500);
> > +
> > +	/*
> > +	 * The sampler won't interpolate the texture correctly along the X axis,
> > +	 * as each RGBA pixel effectively stores two pixels. We thus need to
> > +	 * interpolate manually.
> > +	 *
> > +	 * In integer texture coordinates, the Y values are layed out in the
> > +	 * texture memory as follows:
> > +	 *
> > +	 * ...| Y  U  Y  V | Y  U  Y  V | Y  U  Y  V |...
> > +	 * ...| R  G  B  A | R  G  B  A | R  G  B  A |...
> > +	 *      ^     ^      ^     ^      ^     ^
> > +	 *      |     |      |     |      |     |
> > +	 *     n-1  n-0.5    n   n+0.5   n+1  n+1.5
> > +	 *
> > +	 * For a texture location x in the interval [n, n+1[, sample the left
> 
> s/[n, n+1[/[n, n+1]/

This was on purpose, it means an interval from n to n+1 that includes n
but doesn't include n+1. If x was equal to n+1, it would be in the
interval [n', n'+1[ where n' = n+1 :-)

> > +	 * and right pixels at n and n+1, and interpolate them with
> > +	 *
> > +	 * left.r * (1 - a) + left.b * a	if fract(x) < 0.5
> > +	 * left.b * (1 - a) + right.r * a	if fract(x) >= 0.5
> > +	 *
> > +	 * with a = fract(x * 2) which can also be written
> > +	 *
> > +	 * a = fract(x) * 2			if fract(x) < 0.5
> > +	 * a = fract(x) * 2 - 1			if fract(x) >= 0.5
> > +	 */
> > +	vec2 pos = textureOut;
> > +	float f_x = fract(pos.x / tex_stepx);
> > +
> > +	vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y));
> > +	vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y));
> > +
> > +#if defined(YUV_PATTERN_UYVY)
> > +	float y_left = mix(left.g, left.a, f_x * 2.0);
> > +	float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> > +	vec2 uv = mix(left.rb, right.rb, f_x);
> > +#elif defined(YUV_PATTERN_VYUY)
> > +	float y_left = mix(left.g, left.a, f_x * 2.0);
> > +	float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> > +	vec2 uv = mix(left.br, right.br, f_x);
> > +#elif defined(YUV_PATTERN_YUYV)
> > +	float y_left = mix(left.r, left.b, f_x * 2.0);
> > +	float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> > +	vec2 uv = mix(left.ga, right.ga, f_x);
> > +#elif defined(YUV_PATTERN_YVYU)
> > +	float y_left = mix(left.r, left.b, f_x * 2.0);
> > +	float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> > +	vec2 uv = mix(left.ag, right.ag, f_x);
> > +#else
> > +#error Invalid pattern
> > +#endif
> > +
> > +	float y = mix(y_left, y_right, step(0.5, f_x));
> > +
> > +	vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset);
> > +
> > +	gl_FragColor = vec4(rgb, 1.0);
> > +}
> > diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> > index 7010d8433c9b..857ed9fd5cde 100644
> > --- a/src/qcam/assets/shader/shaders.qrc
> > +++ b/src/qcam/assets/shader/shaders.qrc
> > @@ -4,5 +4,6 @@
> >  	<file>YUV.vert</file>
> >  	<file>YUV_2_planes.frag</file>
> >  	<file>YUV_3_planes.frag</file>
> > +	<file>YUV_packed.frag</file>
> >  </qresource>
> >  </RCC>
> > diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> > index b8a4827267c3..0b5c942658cd 100644
> > --- a/src/qcam/viewfinder_gl.cpp
> > +++ b/src/qcam/viewfinder_gl.cpp
> > @@ -14,12 +14,19 @@
> >  #include <libcamera/formats.h>
> >  
> >  static const QList<libcamera::PixelFormat> supportedFormats{
> > +	/* Packed (single plane) */
> > +	libcamera::formats::UYVY,
> > +	libcamera::formats::VYUY,
> > +	libcamera::formats::YUYV,
> > +	libcamera::formats::YVYU,
> > +	/* Semi planar (two planes) */
> >  	libcamera::formats::NV12,
> >  	libcamera::formats::NV21,
> >  	libcamera::formats::NV16,
> >  	libcamera::formats::NV61,
> >  	libcamera::formats::NV24,
> >  	libcamera::formats::NV42,
> > +	/* Fully planar (three planes) */
> >  	libcamera::formats::YUV420,
> >  	libcamera::formats::YVU420,
> >  };
> > @@ -149,6 +156,22 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> >  		vertSubSample_ = 2;
> >  		fragmentShaderFile_ = ":YUV_3_planes.frag";
> >  		break;
> > +	case libcamera::formats::UYVY:
> > +		fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY");
> > +		fragmentShaderFile_ = ":YUV_packed.frag";
> > +		break;
> > +	case libcamera::formats::VYUY:
> > +		fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY");
> > +		fragmentShaderFile_ = ":YUV_packed.frag";
> > +		break;
> > +	case libcamera::formats::YUYV:
> > +		fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV");
> > +		fragmentShaderFile_ = ":YUV_packed.frag";
> > +		break;
> > +	case libcamera::formats::YVYU:
> > +		fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU");
> > +		fragmentShaderFile_ = ":YUV_packed.frag";
> > +		break;
> >  	default:
> >  		ret = false;
> >  		qWarning() << "[ViewFinderGL]:"
> > @@ -235,6 +258,7 @@ bool ViewFinderGL::createFragmentShader()
> >  	textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
> >  	textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
> >  	textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
> > +	textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx");
> >  
> >  	if (!textureY_.isCreated())
> >  		textureY_.create();
> > @@ -431,6 +455,38 @@ void ViewFinderGL::doRender()
> >  		shaderProgram_.setUniformValue(textureUniformU_, 1);
> >  		break;
> >  
> > +	case libcamera::formats::UYVY:
> > +	case libcamera::formats::VYUY:
> > +	case libcamera::formats::YUYV:
> > +	case libcamera::formats::YVYU:
> > +		/*
> > +		 * Packed YUV formats are stored in a RGBA texture to match the
> > +		 * OpenGL texel size with the 4 bytes repeating pattern in YUV.
> > +		 * The texture width is thus half of the image with.
> > +		 */
> > +		glActiveTexture(GL_TEXTURE0);
> > +		configureTexture(textureY_);
> > +		glTexImage2D(GL_TEXTURE_2D,
> > +			     0,
> > +			     GL_RGBA,
> > +			     size_.width() / 2,
> > +			     size_.height(),
> > +			     0,
> > +			     GL_RGBA,
> > +			     GL_UNSIGNED_BYTE,
> > +			     yuvData_);
> > +		shaderProgram_.setUniformValue(textureUniformY_, 0);
> > +
> > +		/*
> > +		 * The shader needs the step between two texture pixels in the
> > +		 * horizontal direction, expressed in texture coordinate units
> > +		 * ([0, 1]). There are exactly width - 1 steps between the
> > +		 * leftmost and rightmost texels.
> > +		 */
> > +		shaderProgram_.setUniformValue(textureUniformStepX_,
> > +					       1.0f / (size_.width() / 2 - 1));
> > +		break;
> > +
> >  	default:
> >  		break;
> >  	};
> > diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
> > index 53424dc10bc5..ad1e195e45c7 100644
> > --- a/src/qcam/viewfinder_gl.h
> > +++ b/src/qcam/viewfinder_gl.h
> > @@ -79,6 +79,7 @@ private:
> >  	GLuint textureUniformU_;
> >  	GLuint textureUniformV_;
> >  	GLuint textureUniformY_;
> > +	GLuint textureUniformStepX_;
> >  	QOpenGLTexture textureU_;
> >  	QOpenGLTexture textureV_;
> >  	QOpenGLTexture textureY_;