| Message ID | 20250611013245.133785-27-bryan.odonoghue@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series |
|
| Related | show |
On 11/06/2025 02:32, Bryan O'Donoghue wrote: > Extend out the bayer fragment shaders to take 3 x 256 byte inputs as > textures from the CPU. > > We then use an index to the table to recover the colour-corrected values > provided by the SoftIPA thread. > > Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> > --- > .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ > .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- > src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ > src/libcamera/software_isp/debayer_egl.h | 7 +- The fragment shader changes and CPU side class changes should be in individual patches, one patch for the shader, one patch for the CPU. --- bod
Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > Extend out the bayer fragment shaders to take 3 x 256 byte inputs as > textures from the CPU. > > We then use an index to the table to recover the colour-corrected values > provided by the SoftIPA thread. > > Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> > --- > .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ > .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- > src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ > src/libcamera/software_isp/debayer_egl.h | 7 +- > 4 files changed, 169 insertions(+), 23 deletions(-) > > diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag > index 19b13ad0..90bd6457 100644 > --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag > +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag > @@ -65,6 +65,10 @@ uniform vec2 tex_step; > uniform vec2 tex_bayer_first_red; > > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > +uniform mat3 ccm; > > void main(void) > { > @@ -212,5 +216,57 @@ void main(void) > vec3(patterns.y, C, patterns.x) : > vec3(patterns.wz, C)); > > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin Yes. > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); Is it possible to multiply the matrix with the vector directly, something like `ccm * rgb' or `rgb * ccm' (depending on how it fits the transpositions)? > + > +#elif defined(APPLY_RGB_PARAMETERS) Considering the invariant APPLY_CCM_PARAMETERS == !APPLY_RGB_PARAMETERS, would discarding APPLY_RGB_PARAMETERS and using #else instead work better? (If we want to keep the non-CCM case at all.) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; > +#endif > + > gl_FragColor = vec4(rgb, 1.0); > } > diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag > index aa7a1b00..5955c2ea 100644 > --- a/include/libcamera/internal/shaders/bayer_8.frag > +++ b/include/libcamera/internal/shaders/bayer_8.frag > @@ -21,11 +21,17 @@ precision highp float; > > /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ > uniform sampler2D tex_y; > +uniform sampler2D red_param; > +uniform sampler2D green_param; > +uniform sampler2D blue_param; > varying vec4 center; > varying vec4 yCoord; > varying vec4 xCoord; > +uniform mat3 ccm; > > void main(void) { > + vec3 rgb; > + > #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r > > float C = texture2D(tex_y, center.xy).r; // ( 0, 0) > @@ -97,11 +103,65 @@ void main(void) { > PATTERN.xw += kB.xw * B; > PATTERN.xz += kF.xz * F; > > - gl_FragColor.rgb = (alternate.y == 0.0) ? > + rgb = (alternate.y == 0.0) ? > ((alternate.x == 0.0) ? > vec3(C, PATTERN.xy) : > vec3(PATTERN.z, C, PATTERN.w)) : > ((alternate.x == 0.0) ? > vec3(PATTERN.w, C, PATTERN.z) : > vec3(PATTERN.yx, C)); > + > +#if defined(APPLY_CCM_PARAMETERS) > + /* > + * CCM is a 3x3 in the format > + * > + * +--------------+----------------+---------------+ > + * | RedRedGain | RedGreenGain | RedBlueGain | > + * +--------------+----------------+---------------+ > + * | GreenRedGain | GreenGreenGain | GreenBlueGain | > + * +--------------+----------------+---------------+ > + * | BlueRedGain | BlueGreenGain | BlueBlueGain | > + * +--------------+----------------+---------------+ > + * > + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin > + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin > + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > + * > + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); > + * > + * CPU > + * float ccm [] = { > + * RedRedGain, RedGreenGain, RedBlueGain, > + * GreenRedGain, GreenGreenGain, GreenBlueGain, > + * BlueRedGain, BlueGreenGain, BlueBlueGain, > + * }; > + * > + * GPU > + * ccm = { > + * RedRedGain, GreenRedGain, BlueRedGain, > + * RedGreenGain, GreenGreenGain, BlueGreenGain, > + * RedBlueGain, GreenBlueGain, BlueBlueGain, > + * } > + * > + * However the indexing for the mat data-type is column major hence > + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain > + * > + */ > + float rin, gin, bin; > + rin = rgb.r; > + gin = rgb.g; > + bin = rgb.b; > + > + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); > + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); > + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > + > +#elif defined(APPLY_RGB_PARAMETERS) > + /* Apply bayer params */ > + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; > + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; > + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; This shouldn't be all red_param? > +#endif > + > + gl_FragColor.rgb = rgb; > } > diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp > index 3fb15511..824cd6d3 100644 > --- a/src/libcamera/software_isp/debayer_egl.cpp > +++ b/src/libcamera/software_isp/debayer_egl.cpp > @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) > textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); > textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); > textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); > + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); > > textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); > textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); > @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) > << " red_param " << textureUniformRedLookupDataIn_ > << " green_param " << textureUniformGreenLookupDataIn_ > << " blue_param " << textureUniformBlueLookupDataIn_ > + << " ccm " << ccmUniformDataIn_ > << " tex_step " << textureUniformStep_ > << " tex_size " << textureUniformSize_ > << " stride_factor " << textureUniformStrideFactor_ > @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm > break; > }; > > - // Flag to shaders that we have parameter gain tables > - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); > + if (ccmEnabled_) { > + // Run the CCM if available > + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); > + } else { > + // Flag to shaders that we have parameter gain tables > + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); > + } > > if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) > goto compile_fail; > @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, > const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs, > bool ccmEnabled) > { > + GLint maxTextureImageUnits; > + > if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) > return -EINVAL; > > @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, > inputConfig_.stride = inputCfg.stride; > width_ = inputCfg.size.width; > height_ = inputCfg.size.height; > - ccmEnabled_ = ccmEnabled = false; > + ccmEnabled_ = ccmEnabled = true; Looks like a temporary change before ccmEnabled is honoured; but why exactly? > > if (outputCfgs.size() != 1) { > LOG(Debayer, Error) > @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, > glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); > LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; > > - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { > + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { > LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits > << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS > << " try using an identity CCM "; > return -ENODEV; > } > + > // Raw bayer input as texture > eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); > if (!eglImageBayerIn_) > return -ENOMEM; > > - /// RGB correction tables as 2d textures > - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate > - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); > - if (!eglImageRedLookup_) > - return -ENOMEM; > + // Only do the RGB lookup table textures if CCM is disabled > + if (!ccmEnabled_) { > > - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); > - if (!eglImageGreenLookup_) > - return -ENOMEM; > + /// RGB correction tables as 2d textures > + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate > + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); > + if (!eglImageRedLookup_) > + return -ENOMEM; > > - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); > - if (!eglImageBlueLookup_) > - return -ENOMEM; > + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); > + if (!eglImageGreenLookup_) > + return -ENOMEM; > + > + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); > + if (!eglImageBlueLookup_) > + return -ENOMEM; > + } > > // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) > if (gbmSurface_.mapSurface()) > @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) > // To simultaneously sample multiple textures we need to use multiple > // texture units > glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); > - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); > - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); > - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); > + if (!ccmEnabled_) { > + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); > + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); > + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); > + } > > // These values are: > // firstRed = tex_bayer_first_red - bayer_8.vert > @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay > egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); > > // Populate bayer parameters > - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); > - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); > - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); > + if (ccmEnabled_) { > + GLfloat ccm[] = { > + 1, 0, 0, > + 0, 1, 0, > + 0, 0, 1, > + }; > + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); > + } else { > + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); > + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); > + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); > + } > > // Setup the scene > setShaderVariableValues(); > diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h > index c0fc220f..94bc6fc4 100644 > --- a/src/libcamera/software_isp/debayer_egl.h > +++ b/src/libcamera/software_isp/debayer_egl.h > @@ -134,17 +134,22 @@ private: > GLint textureUniformProjMatrix_; > > GLint textureUniformBayerDataIn_; > + > + // These textures will either point to simple RGB gains or to CCM lookup tables > GLint textureUniformRedLookupDataIn_; > GLint textureUniformGreenLookupDataIn_; > GLint textureUniformBlueLookupDataIn_; > > + // Represent per-frame CCM as a uniform vector of floats 3 x 3 > + GLint ccmUniformDataIn_; > + bool ccmEnabled_; > + > Rectangle window_; > std::unique_ptr<SwStatsCpu> stats_; > eGL egl_; > GBM gbmSurface_; > uint32_t width_; > uint32_t height_; > - bool ccmEnabled_; > > GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { > { -1.0f, -1.0f },
On 18/06/2025 13:05, Milan Zamazal wrote: > Bryan O'Donoghue <bryan.odonoghue@linaro.org> writes: > >> Extend out the bayer fragment shaders to take 3 x 256 byte inputs as >> textures from the CPU. >> >> We then use an index to the table to recover the colour-corrected values >> provided by the SoftIPA thread. >> >> Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> >> --- >> .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ >> .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- >> src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ >> src/libcamera/software_isp/debayer_egl.h | 7 +- >> 4 files changed, 169 insertions(+), 23 deletions(-) >> >> diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag >> index 19b13ad0..90bd6457 100644 >> --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag >> +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag >> @@ -65,6 +65,10 @@ uniform vec2 tex_step; >> uniform vec2 tex_bayer_first_red; >> >> uniform sampler2D tex_y; >> +uniform sampler2D red_param; >> +uniform sampler2D green_param; >> +uniform sampler2D blue_param; >> +uniform mat3 ccm; >> >> void main(void) >> { >> @@ -212,5 +216,57 @@ void main(void) >> vec3(patterns.y, C, patterns.x) : >> vec3(patterns.wz, C)); >> >> +#if defined(APPLY_CCM_PARAMETERS) >> + /* >> + * CCM is a 3x3 in the format >> + * >> + * +--------------+----------------+---------------+ >> + * | RedRedGain | RedGreenGain | RedBlueGain | >> + * +--------------+----------------+---------------+ >> + * | GreenRedGain | GreenGreenGain | GreenBlueGain | >> + * +--------------+----------------+---------------+ >> + * | BlueRedGain | BlueGreenGain | BlueBlueGain | >> + * +--------------+----------------+---------------+ >> + * >> + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin >> + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin >> + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin > > Yes. > >> + * >> + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); >> + * >> + * CPU >> + * float ccm [] = { >> + * RedRedGain, RedGreenGain, RedBlueGain, >> + * GreenRedGain, GreenGreenGain, GreenBlueGain, >> + * BlueRedGain, BlueGreenGain, BlueBlueGain, >> + * }; >> + * >> + * GPU >> + * ccm = { >> + * RedRedGain, GreenRedGain, BlueRedGain, >> + * RedGreenGain, GreenGreenGain, BlueGreenGain, >> + * RedBlueGain, GreenBlueGain, BlueBlueGain, >> + * } >> + * >> + * However the indexing for the mat data-type is column major hence >> + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain >> + * >> + */ >> + float rin, gin, bin; >> + rin = rgb.r; >> + gin = rgb.g; >> + bin = rgb.b; >> + >> + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); >> + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); >> + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); > > Is it possible to multiply the matrix with the vector directly, > something like `ccm * rgb' or `rgb * ccm' (depending on how it fits the > transpositions)? That depends on whether glsl "knows what we mean" by way of the type being mat3 - because the in-memory layout is as above not linear. Worth a try. > >> + >> +#elif defined(APPLY_RGB_PARAMETERS) > > Considering the invariant > APPLY_CCM_PARAMETERS == !APPLY_RGB_PARAMETERS, would discarding > APPLY_RGB_PARAMETERS and using #else instead work better? (If we want > to keep the non-CCM case at all.) ah yeah that makes sense > >> + /* Apply bayer params */ >> + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; >> + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; >> + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; >> +#endif >> + >> gl_FragColor = vec4(rgb, 1.0); >> } >> diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag >> index aa7a1b00..5955c2ea 100644 >> --- a/include/libcamera/internal/shaders/bayer_8.frag >> +++ b/include/libcamera/internal/shaders/bayer_8.frag >> @@ -21,11 +21,17 @@ precision highp float; >> >> /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ >> uniform sampler2D tex_y; >> +uniform sampler2D red_param; >> +uniform sampler2D green_param; >> +uniform sampler2D blue_param; >> varying vec4 center; >> varying vec4 yCoord; >> varying vec4 xCoord; >> +uniform mat3 ccm; >> >> void main(void) { >> + vec3 rgb; >> + >> #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r >> >> float C = texture2D(tex_y, center.xy).r; // ( 0, 0) >> @@ -97,11 +103,65 @@ void main(void) { >> PATTERN.xw += kB.xw * B; >> PATTERN.xz += kF.xz * F; >> >> - gl_FragColor.rgb = (alternate.y == 0.0) ? >> + rgb = (alternate.y == 0.0) ? >> ((alternate.x == 0.0) ? >> vec3(C, PATTERN.xy) : >> vec3(PATTERN.z, C, PATTERN.w)) : >> ((alternate.x == 0.0) ? >> vec3(PATTERN.w, C, PATTERN.z) : >> vec3(PATTERN.yx, C)); >> + >> +#if defined(APPLY_CCM_PARAMETERS) >> + /* >> + * CCM is a 3x3 in the format >> + * >> + * +--------------+----------------+---------------+ >> + * | RedRedGain | RedGreenGain | RedBlueGain | >> + * +--------------+----------------+---------------+ >> + * | GreenRedGain | GreenGreenGain | GreenBlueGain | >> + * +--------------+----------------+---------------+ >> + * | BlueRedGain | BlueGreenGain | BlueBlueGain | >> + * +--------------+----------------+---------------+ >> + * >> + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin >> + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin >> + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin >> + * >> + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); >> + * >> + * CPU >> + * float ccm [] = { >> + * RedRedGain, RedGreenGain, RedBlueGain, >> + * GreenRedGain, GreenGreenGain, GreenBlueGain, >> + * BlueRedGain, BlueGreenGain, BlueBlueGain, >> + * }; >> + * >> + * GPU >> + * ccm = { >> + * RedRedGain, GreenRedGain, BlueRedGain, >> + * RedGreenGain, GreenGreenGain, BlueGreenGain, >> + * RedBlueGain, GreenBlueGain, BlueBlueGain, >> + * } >> + * >> + * However the indexing for the mat data-type is column major hence >> + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain >> + * >> + */ >> + float rin, gin, bin; >> + rin = rgb.r; >> + gin = rgb.g; >> + bin = rgb.b; >> + >> + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); >> + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); >> + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); >> + >> +#elif defined(APPLY_RGB_PARAMETERS) >> + /* Apply bayer params */ >> + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; >> + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; >> + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; > > This shouldn't be all red_param? Absolutely. I've been working with the 10/12 bit shader so relying on reviewers/testers for the 8bit shader. Missed this, thanks. > >> +#endif >> + >> + gl_FragColor.rgb = rgb; >> } >> diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp >> index 3fb15511..824cd6d3 100644 >> --- a/src/libcamera/software_isp/debayer_egl.cpp >> +++ b/src/libcamera/software_isp/debayer_egl.cpp >> @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) >> textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); >> textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); >> textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); >> + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); >> >> textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); >> textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); >> @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) >> << " red_param " << textureUniformRedLookupDataIn_ >> << " green_param " << textureUniformGreenLookupDataIn_ >> << " blue_param " << textureUniformBlueLookupDataIn_ >> + << " ccm " << ccmUniformDataIn_ >> << " tex_step " << textureUniformStep_ >> << " tex_size " << textureUniformSize_ >> << " stride_factor " << textureUniformStrideFactor_ >> @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm >> break; >> }; >> >> - // Flag to shaders that we have parameter gain tables >> - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); >> + if (ccmEnabled_) { >> + // Run the CCM if available >> + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); >> + } else { >> + // Flag to shaders that we have parameter gain tables >> + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); >> + } >> >> if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) >> goto compile_fail; >> @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, >> const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs, >> bool ccmEnabled) >> { >> + GLint maxTextureImageUnits; >> + >> if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) >> return -EINVAL; >> >> @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, >> inputConfig_.stride = inputCfg.stride; >> width_ = inputCfg.size.width; >> height_ = inputCfg.size.height; >> - ccmEnabled_ = ccmEnabled = false; >> + ccmEnabled_ = ccmEnabled = true; > > Looks like a temporary change before ccmEnabled is honoured; but why > exactly? Yes, forgot to remove thanks for spotting. --- bod
diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad0..90bd6457 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index aa7a1b00..5955c2ea 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; } diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 3fb15511..824cd6d3 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) << " red_param " << textureUniformRedLookupDataIn_ << " green_param " << textureUniformGreenLookupDataIn_ << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ << " tex_step " << textureUniformStep_ << " tex_size " << textureUniformSize_ << " stride_factor " << textureUniformStrideFactor_ @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm break; }; - // Flag to shaders that we have parameter gain tables - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) goto compile_fail; @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs, bool ccmEnabled) { + GLint maxTextureImageUnits; + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) return -EINVAL; @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = false; + ccmEnabled_ = ccmEnabled = true; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS << " try using an identity CCM "; return -ENODEV; } + // Raw bayer input as texture eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); if (!eglImageBayerIn_) return -ENOMEM; - /// RGB correction tables as 2d textures - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); - if (!eglImageRedLookup_) - return -ENOMEM; + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); - if (!eglImageGreenLookup_) - return -ENOMEM; + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); - if (!eglImageBlueLookup_) - return -ENOMEM; + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) if (gbmSurface_.mapSurface()) @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) // To simultaneously sample multiple textures we need to use multiple // texture units glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } // These values are: // firstRed = tex_bayer_first_red - bayer_8.vert @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); // Populate bayer parameters - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + if (ccmEnabled_) { + GLfloat ccm[] = { + 1, 0, 0, + 0, 1, 0, + 0, 0, 1, + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } // Setup the scene setShaderVariableValues(); diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index c0fc220f..94bc6fc4 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -134,17 +134,22 @@ private: GLint textureUniformProjMatrix_; GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables GLint textureUniformRedLookupDataIn_; GLint textureUniformGreenLookupDataIn_; GLint textureUniformBlueLookupDataIn_; + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + Rectangle window_; std::unique_ptr<SwStatsCpu> stats_; eGL egl_; GBM gbmSurface_; uint32_t width_; uint32_t height_; - bool ccmEnabled_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f },
Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> --- .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 7 +- 4 files changed, 169 insertions(+), 23 deletions(-)