diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp
index 5dca10aee..774f2bf75 100644
--- a/src/libcamera/software_isp/software_isp.cpp
+++ b/src/libcamera/software_isp/software_isp.cpp
@@ -350,9 +350,8 @@ int SoftwareIsp::queueBuffers(uint32_t frame, FrameBuffer *input,
 			return -EINVAL;
 	}
 
-	queuedInputBuffers_.push_back(input);
-
 	for (const auto &[stream, buffer] : outputs) {
+		queuedInputBuffers_.push_back(input);
 		queuedOutputBuffers_.push_back(buffer);
 		process(frame, input, buffer);
 	}