diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp
index acf2d56cddb2..a60415d93705 100644
--- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp
+++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp
@@ -1338,17 +1338,16 @@ void RPiCameraData::embeddedComplete(uint32_t bufferId)
 
 void RPiCameraData::setIspControls(const ControlList &controls)
 {
-	ControlList ctrls = controls;
-
-	Span<const uint8_t> s =
-		ctrls.get(V4L2_CID_USER_BCM2835_ISP_LENS_SHADING).data();
-	bcm2835_isp_lens_shading ls =
-		*reinterpret_cast<const bcm2835_isp_lens_shading *>(s.data());
-	ls.dmabuf = lsTable_.fd();
-
-	ControlValue c(Span<const uint8_t>{ reinterpret_cast<uint8_t *>(&ls),
-					    sizeof(ls) });
-	ctrls.set(V4L2_CID_USER_BCM2835_ISP_LENS_SHADING, c);
+	ControlList ctrls = std::move(controls);
+
+	if (ctrls.contains(V4L2_CID_USER_BCM2835_ISP_LENS_SHADING)) {
+		ControlValue &value =
+			const_cast<ControlValue &>(ctrls.get(V4L2_CID_USER_BCM2835_ISP_LENS_SHADING));
+		Span<uint8_t> s = value.data();
+		bcm2835_isp_lens_shading *ls =
+			reinterpret_cast<bcm2835_isp_lens_shading *>(s.data());
+		ls->dmabuf = lsTable_.fd();
+	}
 
 	isp_[Isp::Input].dev()->setControls(&ctrls);
 	handleState();