diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp
index 28e2a360e7e6d5071e7b97c2bffc60478d245a60..7bee8f0681dd5cedcd81950ed49f611b514e763d 100644
--- a/src/libcamera/software_isp/software_isp.cpp
+++ b/src/libcamera/software_isp/software_isp.cpp
@@ -159,8 +159,6 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor,
 					    metadataReady.emit(frame, metadata);
 				    });
 	ipa_->setSensorControls.connect(this, &SoftwareIsp::setSensorCtrls);
-
-	debayer_->moveToThread(&ispWorkerThread_);
 }
 
 SoftwareIsp::~SoftwareIsp()
@@ -262,6 +260,9 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg,
 	if (ret < 0)
 		return ret;
 
+	debayer_->moveToThread(&ispWorkerThread_);
+	ispWorkerThread_.start();
+
 	return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_);
 }
 
@@ -343,7 +344,6 @@ int SoftwareIsp::start()
 	if (ret)
 		return ret;
 
-	ispWorkerThread_.start();
 	return 0;
 }