diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp
index af08ab10..d3831474 100644
--- a/src/libcamera/software_isp/debayer_cpu.cpp
+++ b/src/libcamera/software_isp/debayer_cpu.cpp
@@ -51,7 +51,8 @@ DebayerCpu::DebayerCpu(std::unique_ptr<SwStatsCpu> stats,
 
 	/* Initialize color lookup tables */
 	for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++)
-		red_[i] = green_[i] = blue_[i] = i;
+		initColorLookupTable_[i] = i;
+	red_ = green_ = blue_ = &initColorLookupTable_;
 
 	paramsBuffers_ = std::map<unsigned int, DebayerParams *>();
 
@@ -80,12 +81,12 @@ DebayerCpu::~DebayerCpu() = default;
  * GBG
  * RGR
  */
-#define BGGR_BGR888(p, n, div)                                                                \
-	*dst++ = blue_[curr[x] / (div)];                                                      \
-	*dst++ = green_[(prev[x] + curr[x - p] + curr[x + n] + next[x]) / (4 * (div))];       \
-	*dst++ = red_[(prev[x - p] + prev[x + n] + next[x - p] + next[x + n]) / (4 * (div))]; \
-	if constexpr (addAlphaByte)                                                           \
-		*dst++ = 255;                                                                 \
+#define BGGR_BGR888(p, n, div)                                                                   \
+	*dst++ = (*blue_)[curr[x] / (div)];                                                      \
+	*dst++ = (*green_)[(prev[x] + curr[x - p] + curr[x + n] + next[x]) / (4 * (div))];       \
+	*dst++ = (*red_)[(prev[x - p] + prev[x + n] + next[x - p] + next[x + n]) / (4 * (div))]; \
+	if constexpr (addAlphaByte)                                                              \
+		*dst++ = 255;                                                                    \
 	x++;
 
 /*
@@ -93,12 +94,12 @@ DebayerCpu::~DebayerCpu() = default;
  * RGR
  * GBG
  */
-#define GRBG_BGR888(p, n, div)                                    \
-	*dst++ = blue_[(prev[x] + next[x]) / (2 * (div))];        \
-	*dst++ = green_[curr[x] / (div)];                         \
-	*dst++ = red_[(curr[x - p] + curr[x + n]) / (2 * (div))]; \
-	if constexpr (addAlphaByte)                               \
-		*dst++ = 255;                                     \
+#define GRBG_BGR888(p, n, div)                                       \
+	*dst++ = (*blue_)[(prev[x] + next[x]) / (2 * (div))];        \
+	*dst++ = (*green_)[curr[x] / (div)];                         \
+	*dst++ = (*red_)[(curr[x - p] + curr[x + n]) / (2 * (div))]; \
+	if constexpr (addAlphaByte)                                  \
+		*dst++ = 255;                                        \
 	x++;
 
 /*
@@ -106,12 +107,12 @@ DebayerCpu::~DebayerCpu() = default;
  * BGB
  * GRG
  */
-#define GBRG_BGR888(p, n, div)                                     \
-	*dst++ = blue_[(curr[x - p] + curr[x + n]) / (2 * (div))]; \
-	*dst++ = green_[curr[x] / (div)];                          \
-	*dst++ = red_[(prev[x] + next[x]) / (2 * (div))];          \
-	if constexpr (addAlphaByte)                                \
-		*dst++ = 255;                                      \
+#define GBRG_BGR888(p, n, div)                                        \
+	*dst++ = (*blue_)[(curr[x - p] + curr[x + n]) / (2 * (div))]; \
+	*dst++ = (*green_)[curr[x] / (div)];                          \
+	*dst++ = (*red_)[(prev[x] + next[x]) / (2 * (div))];          \
+	if constexpr (addAlphaByte)                                   \
+		*dst++ = 255;                                         \
 	x++;
 
 /*
@@ -119,12 +120,12 @@ DebayerCpu::~DebayerCpu() = default;
  * GRG
  * BGB
  */
-#define RGGB_BGR888(p, n, div)                                                                 \
-	*dst++ = blue_[(prev[x - p] + prev[x + n] + next[x - p] + next[x + n]) / (4 * (div))]; \
-	*dst++ = green_[(prev[x] + curr[x - p] + curr[x + n] + next[x]) / (4 * (div))];        \
-	*dst++ = red_[curr[x] / (div)];                                                        \
-	if constexpr (addAlphaByte)                                                            \
-		*dst++ = 255;                                                                  \
+#define RGGB_BGR888(p, n, div)                                                                    \
+	*dst++ = (*blue_)[(prev[x - p] + prev[x + n] + next[x - p] + next[x + n]) / (4 * (div))]; \
+	*dst++ = (*green_)[(prev[x] + curr[x - p] + curr[x + n] + next[x]) / (4 * (div))];        \
+	*dst++ = (*red_)[curr[x] / (div)];                                                        \
+	if constexpr (addAlphaByte)                                                               \
+		*dst++ = 255;                                                                     \
 	x++;
 
 template<bool addAlphaByte>
@@ -752,12 +753,10 @@ void DebayerCpu::process(uint32_t frame,
 	}
 
 	DebayerParams *params = paramsBuffers_.at(paramsBufferId);
-	releaseIspParams.emit(paramsBufferId);
 
-	/* \todo Avoid copying here. */
-	green_ = params->green;
-	red_ = swapRedBlueGains_ ? params->blue : params->red;
-	blue_ = swapRedBlueGains_ ? params->red : params->blue;
+	green_ = &params->green;
+	red_ = swapRedBlueGains_ ? &params->blue : &params->red;
+	blue_ = swapRedBlueGains_ ? &params->red : &params->blue;
 
 	/* Copy metadata from the input buffer */
 	FrameMetadata &metadata = output->_d()->metadata();
@@ -779,6 +778,7 @@ void DebayerCpu::process(uint32_t frame,
 		process2(in.planes()[0].data(), out.planes()[0].data());
 	else
 		process4(in.planes()[0].data(), out.planes()[0].data());
+	releaseIspParams.emit(paramsBufferId);
 
 	metadata.planes()[0].bytesused = out.planes()[0].size();
 
diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h
index 324fc6c0..f25520be 100644
--- a/src/libcamera/software_isp/debayer_cpu.h
+++ b/src/libcamera/software_isp/debayer_cpu.h
@@ -139,9 +139,10 @@ private:
 	/* Max. supported Bayer pattern height is 4, debayering this requires 5 lines */
 	static constexpr unsigned int kMaxLineBuffers = 5;
 
-	DebayerParams::ColorLookupTable red_;
-	DebayerParams::ColorLookupTable green_;
-	DebayerParams::ColorLookupTable blue_;
+	DebayerParams::ColorLookupTable initColorLookupTable_;
+	DebayerParams::ColorLookupTable *red_;
+	DebayerParams::ColorLookupTable *green_;
+	DebayerParams::ColorLookupTable *blue_;
 	debayerFn debayer0_;
 	debayerFn debayer1_;
 	debayerFn debayer2_;