@@ -65,6 +65,10 @@ uniform vec2 tex_step;
uniform vec2 tex_bayer_first_red;
uniform sampler2D tex_y;
+uniform sampler2D red_param;
+uniform sampler2D green_param;
+uniform sampler2D blue_param;
+uniform mat3 ccm;
void main(void)
{
@@ -212,5 +216,57 @@ void main(void)
vec3(patterns.y, C, patterns.x) :
vec3(patterns.wz, C));
+#if defined(APPLY_CCM_PARAMETERS)
+ /*
+ * CCM is a 3x3 in the format
+ *
+ * +--------------+----------------+---------------+
+ * | RedRedGain | RedGreenGain | RedBlueGain |
+ * +--------------+----------------+---------------+
+ * | GreenRedGain | GreenGreenGain | GreenBlueGain |
+ * +--------------+----------------+---------------+
+ * | BlueRedGain | BlueGreenGain | BlueBlueGain |
+ * +--------------+----------------+---------------+
+ *
+ * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin
+ * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin
+ * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin
+ *
+ * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm);
+ *
+ * CPU
+ * float ccm [] = {
+ * RedRedGain, RedGreenGain, RedBlueGain,
+ * GreenRedGain, GreenGreenGain, GreenBlueGain,
+ * BlueRedGain, BlueGreenGain, BlueBlueGain,
+ * };
+ *
+ * GPU
+ * ccm = {
+ * RedRedGain, GreenRedGain, BlueRedGain,
+ * RedGreenGain, GreenGreenGain, BlueGreenGain,
+ * RedBlueGain, GreenBlueGain, BlueBlueGain,
+ * }
+ *
+ * However the indexing for the mat data-type is column major hence
+ * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain
+ *
+ */
+ float rin, gin, bin;
+ rin = rgb.r;
+ gin = rgb.g;
+ bin = rgb.b;
+
+ rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]);
+ rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]);
+ rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]);
+
+#elif defined(APPLY_RGB_PARAMETERS)
+ /* Apply bayer params */
+ rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r;
+ rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g;
+ rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b;
+#endif
+
gl_FragColor = vec4(rgb, 1.0);
}
@@ -21,11 +21,17 @@ precision highp float;
/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/
uniform sampler2D tex_y;
+uniform sampler2D red_param;
+uniform sampler2D green_param;
+uniform sampler2D blue_param;
varying vec4 center;
varying vec4 yCoord;
varying vec4 xCoord;
+uniform mat3 ccm;
void main(void) {
+ vec3 rgb;
+
#define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
float C = texture2D(tex_y, center.xy).r; // ( 0, 0)
@@ -97,11 +103,65 @@ void main(void) {
PATTERN.xw += kB.xw * B;
PATTERN.xz += kF.xz * F;
- gl_FragColor.rgb = (alternate.y == 0.0) ?
+ rgb = (alternate.y == 0.0) ?
((alternate.x == 0.0) ?
vec3(C, PATTERN.xy) :
vec3(PATTERN.z, C, PATTERN.w)) :
((alternate.x == 0.0) ?
vec3(PATTERN.w, C, PATTERN.z) :
vec3(PATTERN.yx, C));
+
+#if defined(APPLY_CCM_PARAMETERS)
+ /*
+ * CCM is a 3x3 in the format
+ *
+ * +--------------+----------------+---------------+
+ * | RedRedGain | RedGreenGain | RedBlueGain |
+ * +--------------+----------------+---------------+
+ * | GreenRedGain | GreenGreenGain | GreenBlueGain |
+ * +--------------+----------------+---------------+
+ * | BlueRedGain | BlueGreenGain | BlueBlueGain |
+ * +--------------+----------------+---------------+
+ *
+ * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin
+ * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin
+ * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin
+ *
+ * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm);
+ *
+ * CPU
+ * float ccm [] = {
+ * RedRedGain, RedGreenGain, RedBlueGain,
+ * GreenRedGain, GreenGreenGain, GreenBlueGain,
+ * BlueRedGain, BlueGreenGain, BlueBlueGain,
+ * };
+ *
+ * GPU
+ * ccm = {
+ * RedRedGain, GreenRedGain, BlueRedGain,
+ * RedGreenGain, GreenGreenGain, BlueGreenGain,
+ * RedBlueGain, GreenBlueGain, BlueBlueGain,
+ * }
+ *
+ * However the indexing for the mat data-type is column major hence
+ * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain
+ *
+ */
+ float rin, gin, bin;
+ rin = rgb.r;
+ gin = rgb.g;
+ bin = rgb.b;
+
+ rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]);
+ rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]);
+ rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]);
+
+#elif defined(APPLY_RGB_PARAMETERS)
+ /* Apply bayer params */
+ rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r;
+ rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g;
+ rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b;
+#endif
+
+ gl_FragColor.rgb = rgb;
}
@@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void)
textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param");
textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param");
textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param");
+ ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm");
textureUniformStep_ = glGetUniformLocation(programId_, "tex_step");
textureUniformSize_ = glGetUniformLocation(programId_, "tex_size");
@@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void)
<< " red_param " << textureUniformRedLookupDataIn_
<< " green_param " << textureUniformGreenLookupDataIn_
<< " blue_param " << textureUniformBlueLookupDataIn_
+ << " ccm " << ccmUniformDataIn_
<< " tex_step " << textureUniformStep_
<< " tex_size " << textureUniformSize_
<< " stride_factor " << textureUniformStrideFactor_
@@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
break;
};
- // Flag to shaders that we have parameter gain tables
- egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS");
+ if (ccmEnabled_) {
+ // Run the CCM if available
+ egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS");
+ } else {
+ // Flag to shaders that we have parameter gain tables
+ egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS");
+ }
if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv))
goto compile_fail;
@@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs,
bool ccmEnabled)
{
+ GLint maxTextureImageUnits;
+
if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0)
return -EINVAL;
@@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
inputConfig_.stride = inputCfg.stride;
width_ = inputCfg.size.width;
height_ = inputCfg.size.height;
- ccmEnabled_ = ccmEnabled = false;
+ ccmEnabled_ = ccmEnabled = true;
if (outputCfgs.size() != 1) {
LOG(Debayer, Error)
@@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits);
LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits;
- if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) {
+ if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) {
LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits
<< " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS
<< " try using an identity CCM ";
return -ENODEV;
}
+
// Raw bayer input as texture
eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0);
if (!eglImageBayerIn_)
return -ENOMEM;
- /// RGB correction tables as 2d textures
- // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate
- eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1);
- if (!eglImageRedLookup_)
- return -ENOMEM;
+ // Only do the RGB lookup table textures if CCM is disabled
+ if (!ccmEnabled_) {
- eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2);
- if (!eglImageGreenLookup_)
- return -ENOMEM;
+ /// RGB correction tables as 2d textures
+ // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate
+ eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1);
+ if (!eglImageRedLookup_)
+ return -ENOMEM;
- eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3);
- if (!eglImageBlueLookup_)
- return -ENOMEM;
+ eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2);
+ if (!eglImageGreenLookup_)
+ return -ENOMEM;
+
+ eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3);
+ if (!eglImageBlueLookup_)
+ return -ENOMEM;
+ }
// Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO)
if (gbmSurface_.mapSurface())
@@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void)
// To simultaneously sample multiple textures we need to use multiple
// texture units
glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_);
- glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_);
- glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_);
- glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_);
+ if (!ccmEnabled_) {
+ glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_);
+ glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_);
+ glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_);
+ }
// These values are:
// firstRed = tex_bayer_first_red - bayer_8.vert
@@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay
egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data());
// Populate bayer parameters
- egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red);
- egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green);
- egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue);
+ if (ccmEnabled_) {
+ GLfloat ccm[] = {
+ 1, 0, 0,
+ 0, 1, 0,
+ 0, 0, 1,
+ };
+ glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm);
+ } else {
+ egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red);
+ egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green);
+ egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue);
+ }
// Setup the scene
setShaderVariableValues();
@@ -134,17 +134,22 @@ private:
GLint textureUniformProjMatrix_;
GLint textureUniformBayerDataIn_;
+
+ // These textures will either point to simple RGB gains or to CCM lookup tables
GLint textureUniformRedLookupDataIn_;
GLint textureUniformGreenLookupDataIn_;
GLint textureUniformBlueLookupDataIn_;
+ // Represent per-frame CCM as a uniform vector of floats 3 x 3
+ GLint ccmUniformDataIn_;
+ bool ccmEnabled_;
+
Rectangle window_;
std::unique_ptr<SwStatsCpu> stats_;
eGL egl_;
GBM gbmSurface_;
uint32_t width_;
uint32_t height_;
- bool ccmEnabled_;
GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = {
{ -1.0f, -1.0f },
Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org> --- .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 7 +- 4 files changed, 169 insertions(+), 23 deletions(-)