| Message ID | 20251015012251.17508-30-bryan.odonoghue@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series |
|
| Related | show |
Hi Bryan, Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > Extend out the bayer fragment shaders to take 3 x 256 byte inputs as > textures from the CPU. > > We then use an index to the table to recover the colour-corrected values > provided by the SoftIPA thread. > > Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> > --- > .../internal/shaders/bayer_1x_packed.frag | 56 +++++++++++++++++ > .../internal/shaders/bayer_unpacked.frag | 62 ++++++++++++++++++- > 2 files changed, 117 insertions(+), 1 deletion(-) > > diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag > index 19b13ad0..90bd6457 100644 > --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag > +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag > @@ -65,6 +65,10 @@ uniform vec2 tex_step; > uniform vec2 tex_bayer_first_red; > > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > +uniform mat3 ccm; > > void main(void) > { > @@ -212,5 +216,57 @@ void main(void) > vec3(patterns.y, C, patterns.x) : > vec3(patterns.wz, C)); > > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ The ordering of the values is tricky. I performed some testing and it seems to work as expected with GPU ISP. Reviewed-by: Milan Zamazal <mzamazal@redhat.com> Unrelated to this patch and branch, we have some problem with CPU ISP CCM. The matrix [ 0, 1, 0, 0, 0, 0, 0, 0, 0] produces a blue rather than red picture in my environment (unlike when 1 is in other positions of the first row), which is weird. I'll look at it. > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; > +#endif > + > gl_FragColor = vec4(rgb, 1.0); > } > diff --git a/include/libcamera/internal/shaders/bayer_unpacked.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag > index aa7a1b00..5955c2ea 100644 > --- a/include/libcamera/internal/shaders/bayer_unpacked.frag > +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag > @@ -21,11 +21,17 @@ precision highp float; > > /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > varying vec4 center; > varying vec4 yCoord; > varying vec4 xCoord; > +uniform mat3 ccm; > > void main(void) { > + vec3 rgb; > + > #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r > > float C = texture2D(tex_y, center.xy).r; // ( 0, 0) > @@ -97,11 +103,65 @@ void main(void) { > PATTERN.xw += kB.xw * B; > PATTERN.xz += kF.xz * F; > > - gl_FragColor.rgb = (alternate.y == 0.0) ? > + rgb = (alternate.y == 0.0) ? > ((alternate.x == 0.0) ? > vec3(C, PATTERN.xy) : > vec3(PATTERN.z, C, PATTERN.w)) : > ((alternate.x == 0.0) ? > vec3(PATTERN.w, C, PATTERN.z) : > vec3(PATTERN.yx, C)); > + > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; > +#endif > + > + gl_FragColor.rgb = rgb; > }
Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > Extend out the bayer fragment shaders to take 3 x 256 byte inputs as > textures from the CPU. > > We then use an index to the table to recover the colour-corrected values > provided by the SoftIPA thread. > > Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> > --- > .../internal/shaders/bayer_1x_packed.frag | 56 +++++++++++++++++ > .../internal/shaders/bayer_unpacked.frag | 62 ++++++++++++++++++- > 2 files changed, 117 insertions(+), 1 deletion(-) > > diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag > index 19b13ad0..90bd6457 100644 > --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag > +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag > @@ -65,6 +65,10 @@ uniform vec2 tex_step; > uniform vec2 tex_bayer_first_red; > > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > +uniform mat3 ccm; > > void main(void) > { > @@ -212,5 +216,57 @@ void main(void) > vec3(patterns.y, C, patterns.x) : > vec3(patterns.wz, C)); > > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); I think black level must be subtracted and gamma applied here, e.g. rgb = rgb - vec3(blackLevel); // blackLevel must be passed from the IPA rgb = pow(rgb, vec3(gamma)); // gamma = 0.5 in CPU ISP See lut.cpp, where with CCM the gamma lookup table is passed separately from the CCM lookup tables to debayer_cpu.cpp and incorporates the black level correction. With these two changes + the awb change in the other patch, APPLY_CCM_PARAMETERS output seems to be similar to APPLY_RGB_PARAMETERS output in my environment. I'm not sure I'm 100% correct and it needs a bit more testing but I think these are the basic ideas what fixes are needed for the CCM output. The same for the other shader. > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; > +#endif > + > gl_FragColor = vec4(rgb, 1.0); > } > diff --git a/include/libcamera/internal/shaders/bayer_unpacked.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag > index aa7a1b00..5955c2ea 100644 > --- a/include/libcamera/internal/shaders/bayer_unpacked.frag > +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag > @@ -21,11 +21,17 @@ precision highp float; > > /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > varying vec4 center; > varying vec4 yCoord; > varying vec4 xCoord; > +uniform mat3 ccm; > > void main(void) { > + vec3 rgb; > + > #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r > > float C = texture2D(tex_y, center.xy).r; // ( 0, 0) > @@ -97,11 +103,65 @@ void main(void) { > PATTERN.xw += kB.xw * B; > PATTERN.xz += kF.xz * F; > > - gl_FragColor.rgb = (alternate.y == 0.0) ? > + rgb = (alternate.y == 0.0) ? > ((alternate.x == 0.0) ? > vec3(C, PATTERN.xy) : > vec3(PATTERN.z, C, PATTERN.w)) : > ((alternate.x == 0.0) ? > vec3(PATTERN.w, C, PATTERN.z) : > vec3(PATTERN.yx, C)); > + > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; > +#endif > + > + gl_FragColor.rgb = rgb; > }
Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > Extend out the bayer fragment shaders to take 3 x 256 byte inputs as > textures from the CPU. > > We then use an index to the table to recover the colour-corrected values > provided by the SoftIPA thread. > > Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> > --- > .../internal/shaders/bayer_1x_packed.frag | 56 +++++++++++++++++ > .../internal/shaders/bayer_unpacked.frag | 62 ++++++++++++++++++- > 2 files changed, 117 insertions(+), 1 deletion(-) > > diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag > index 19b13ad0..90bd6457 100644 > --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag > +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag > @@ -65,6 +65,10 @@ uniform vec2 tex_step; > uniform vec2 tex_bayer_first_red; > > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > +uniform mat3 ccm; > > void main(void) > { > @@ -212,5 +216,57 @@ void main(void) > vec3(patterns.y, C, patterns.x) : > vec3(patterns.wz, C)); > > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; > +#endif > + > gl_FragColor = vec4(rgb, 1.0); > } > diff --git a/include/libcamera/internal/shaders/bayer_unpacked.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag > index aa7a1b00..5955c2ea 100644 > --- a/include/libcamera/internal/shaders/bayer_unpacked.frag > +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag > @@ -21,11 +21,17 @@ precision highp float; > > /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > varying vec4 center; > varying vec4 yCoord; > varying vec4 xCoord; > +uniform mat3 ccm; > > void main(void) { > + vec3 rgb; > + > #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r > > float C = texture2D(tex_y, center.xy).r; // ( 0, 0) > @@ -97,11 +103,65 @@ void main(void) { > PATTERN.xw += kB.xw * B; > PATTERN.xz += kF.xz * F; > > - gl_FragColor.rgb = (alternate.y == 0.0) ? > + rgb = (alternate.y == 0.0) ? > ((alternate.x == 0.0) ? > vec3(C, PATTERN.xy) : > vec3(PATTERN.z, C, PATTERN.w)) : > ((alternate.x == 0.0) ? > vec3(PATTERN.w, C, PATTERN.z) : > vec3(PATTERN.yx, C)); > + > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; All the lookups are mistakenly on red_param rather than red/green/blue_param here. (Commented previously on v1 by mistake.) > +#endif > + > + gl_FragColor.rgb = rgb; > }
On 18/10/2025 11:56, Milan Zamazal wrote: >> +#elif defined(APPLY_RGB_PARAMETERS) >> + /* Apply bayer params */ >> + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; >> + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; >> + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; > All the lookups are mistakenly on red_param rather than > red/green/blue_param here. > > (Commented previously on v1 by mistake.) And I fixed that... Obviously managed to not commit this change ... :( --- bod
Milan Zamazal <mzamazal@redhat.com> writes: > Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > >> Extend out the bayer fragment shaders to take 3 x 256 byte inputs as >> textures from the CPU. >> >> We then use an index to the table to recover the colour-corrected values >> provided by the SoftIPA thread. >> >> Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> >> --- >> .../internal/shaders/bayer_1x_packed.frag | 56 +++++++++++++++++ >> .../internal/shaders/bayer_unpacked.frag | 62 ++++++++++++++++++- >> 2 files changed, 117 insertions(+), 1 deletion(-) >> >> diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag >> index 19b13ad0..90bd6457 100644 >> --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag >> +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag >> @@ -65,6 +65,10 @@ uniform vec2 tex_step; >> uniform vec2 tex_bayer_first_red; >> >> uniform sampler2D tex_y; >> +uniform sampler2D red_param; >> +uniform sampler2D green_param; >> +uniform sampler2D blue_param; >> +uniform mat3 ccm; >> >> void main(void) >> { >> @@ -212,5 +216,57 @@ void main(void) >> vec3(patterns.y, C, patterns.x) : >> vec3(patterns.wz, C)); >> >> +#if defined(APPLY_CCM_PARAMETERS) >> + /* >> + * CCM is a 3x3 in the format >> + * >> + * +--------------+----------------+---------------+ >> + * | RedRedGain | RedGreenGain | RedBlueGain | >> + * +--------------+----------------+---------------+ >> + * | GreenRedGain | GreenGreenGain | GreenBlueGain | >> + * +--------------+----------------+---------------+ >> + * | BlueRedGain | BlueGreenGain | BlueBlueGain | >> + * +--------------+----------------+---------------+ >> + * >> + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin >> + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin >> + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin >> + * >> + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); >> + * >> + * CPU >> + * float ccm [] = { >> + * RedRedGain, RedGreenGain, RedBlueGain, >> + * GreenRedGain, GreenGreenGain, GreenBlueGain, >> + * BlueRedGain, BlueGreenGain, BlueBlueGain, >> + * }; >> + * >> + * GPU >> + * ccm = { >> + * RedRedGain, GreenRedGain, BlueRedGain, >> + * RedGreenGain, GreenGreenGain, BlueGreenGain, >> + * RedBlueGain, GreenBlueGain, BlueBlueGain, >> + * } >> + * >> + * However the indexing for the mat data-type is column major hence >> + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain >> + * >> + */ >> + float rin, gin, bin; >> + rin = rgb.r; >> + gin = rgb.g; >> + bin = rgb.b; >> + >> + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); >> + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); >> + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > > I think black level must be subtracted and gamma applied here, e.g. > > rgb = rgb - vec3(blackLevel); // blackLevel must be passed from the IPA > rgb = pow(rgb, vec3(gamma)); // gamma = 0.5 in CPU ISP > > See lut.cpp, where with CCM the gamma lookup table is passed separately > from the CCM lookup tables to debayer_cpu.cpp and incorporates the black > level correction. Another thing missing is applying the contrast control with CCM, it is available but no-op. See Lut::updateGammaTable for how contrast is computed in CPU ISP, together with gamma. (Without CCM, it's incorporated in the lookup tables and it works.) > With these two changes + the awb change in the other patch, > APPLY_CCM_PARAMETERS output seems to be similar to APPLY_RGB_PARAMETERS > output in my environment. I'm not sure I'm 100% correct and it needs a > bit more testing but I think these are the basic ideas what fixes are > needed for the CCM output. > > The same for the other shader. > >> + >> +#elif defined(APPLY_RGB_PARAMETERS) >> + /* Apply bayer params */ >> + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; >> + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; >> + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; >> +#endif >> + >> gl_FragColor = vec4(rgb, 1.0); >> } >> diff --git a/include/libcamera/internal/shaders/bayer_unpacked.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag >> index aa7a1b00..5955c2ea 100644 >> --- a/include/libcamera/internal/shaders/bayer_unpacked.frag >> +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag >> @@ -21,11 +21,17 @@ precision highp float; >> >> /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ >> uniform sampler2D tex_y; >> +uniform sampler2D red_param; >> +uniform sampler2D green_param; >> +uniform sampler2D blue_param; >> varying vec4 center; >> varying vec4 yCoord; >> varying vec4 xCoord; >> +uniform mat3 ccm; >> >> void main(void) { >> + vec3 rgb; >> + >> #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r >> >> float C = texture2D(tex_y, center.xy).r; // ( 0, 0) >> @@ -97,11 +103,65 @@ void main(void) { >> PATTERN.xw += kB.xw * B; >> PATTERN.xz += kF.xz * F; >> >> - gl_FragColor.rgb = (alternate.y == 0.0) ? >> + rgb = (alternate.y == 0.0) ? >> ((alternate.x == 0.0) ? >> vec3(C, PATTERN.xy) : >> vec3(PATTERN.z, C, PATTERN.w)) : >> ((alternate.x == 0.0) ? >> vec3(PATTERN.w, C, PATTERN.z) : >> vec3(PATTERN.yx, C)); >> + >> +#if defined(APPLY_CCM_PARAMETERS) >> + /* >> + * CCM is a 3x3 in the format >> + * >> + * +--------------+----------------+---------------+ >> + * | RedRedGain | RedGreenGain | RedBlueGain | >> + * +--------------+----------------+---------------+ >> + * | GreenRedGain | GreenGreenGain | GreenBlueGain | >> + * +--------------+----------------+---------------+ >> + * | BlueRedGain | BlueGreenGain | BlueBlueGain | >> + * +--------------+----------------+---------------+ >> + * >> + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin >> + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin >> + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin >> + * >> + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); >> + * >> + * CPU >> + * float ccm [] = { >> + * RedRedGain, RedGreenGain, RedBlueGain, >> + * GreenRedGain, GreenGreenGain, GreenBlueGain, >> + * BlueRedGain, BlueGreenGain, BlueBlueGain, >> + * }; >> + * >> + * GPU >> + * ccm = { >> + * RedRedGain, GreenRedGain, BlueRedGain, >> + * RedGreenGain, GreenGreenGain, BlueGreenGain, >> + * RedBlueGain, GreenBlueGain, BlueBlueGain, >> + * } >> + * >> + * However the indexing for the mat data-type is column major hence >> + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain >> + * >> + */ >> + float rin, gin, bin; >> + rin = rgb.r; >> + gin = rgb.g; >> + bin = rgb.b; >> + >> + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); >> + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); >> + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); >> + >> +#elif defined(APPLY_RGB_PARAMETERS) >> + /* Apply bayer params */ >> + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; >> + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; >> + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; >> +#endif >> + >> + gl_FragColor.rgb = rgb; >> }
On 17/10/2025 19:53, Milan Zamazal wrote: > rgb = rgb - vec3(blackLevel); // blackLevel must be passed from the IPA So my best guess for that to pass here is context.activeState.gamma.blackLevel / 256 Since this is a unit8 we want to normalise it to between 0.0f and 1.0f > rgb = pow(rgb, vec3(gamma)); // gamma = 0.5 in CPU ISP Looking at the code in lut.cpp it seems to me when the gpuisp is running we should not calculate at least some of these lookup tables you mentioned. Seems like wasted cycles, I don't know the code as well as you but. void Lut::updateGammaTable(IPAContext &context) { /* The loop setting this */ gammaTable[i] = UINT8_MAX * std::pow(normalized, context.configuration.gamma); } void Lut::prepare(IPAContext &context, [[maybe_unused]] const uint32_t frame, IPAFrameContext &frameContext, DebayerParams *params) { /* And this assignment in the containing loop */ params->gammaLut[i] = gammaTable[i / div]; } can be avoided iff GPUISP is active. --- bod
Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > On 17/10/2025 19:53, Milan Zamazal wrote: >> rgb = rgb - vec3(blackLevel); // blackLevel must be passed from the IPA > > So my best guess for that to pass here is context.activeState.gamma.blackLevel / 256 context.activeState.blc.level / 256.0 context.activeState.gamma.blackLevel is used only to check for change of the gamma table parameters and is not set if the gamma table is not computed. > Since this is a unit8 we want to normalise it to between 0.0f and 1.0f >> rgb = pow(rgb, vec3(gamma)); // gamma = 0.5 in CPU ISP > > Looking at the code in lut.cpp it seems to me when the gpuisp is running we should not calculate at least some of > these lookup tables you mentioned. > > Seems like wasted cycles, I don't know the code as well as you but. > > void Lut::updateGammaTable(IPAContext &context) > { > /* The loop setting this */ > gammaTable[i] = UINT8_MAX * > std::pow(normalized, context.configuration.gamma); > } > > void Lut::prepare(IPAContext &context, > [[maybe_unused]] const uint32_t frame, > IPAFrameContext &frameContext, > DebayerParams *params) > { > /* And this assignment in the containing loop */ > params->gammaLut[i] = gammaTable[i / div]; > } > > can be avoided iff GPUISP is active. Yes.
diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad0..90bd6457 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_unpacked.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag index aa7a1b00..5955c2ea 100644 --- a/include/libcamera/internal/shaders/bayer_unpacked.frag +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; }
Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> --- .../internal/shaders/bayer_1x_packed.frag | 56 +++++++++++++++++ .../internal/shaders/bayer_unpacked.frag | 62 ++++++++++++++++++- 2 files changed, 117 insertions(+), 1 deletion(-)