diff --git a/src/libcamera/pipeline/simple/converter.cpp b/src/libcamera/pipeline/simple/converter.cpp
index 550b2bcfb001..6b3249ea92b0 100644
--- a/src/libcamera/pipeline/simple/converter.cpp
+++ b/src/libcamera/pipeline/simple/converter.cpp
@@ -195,6 +195,9 @@ int SimpleConverter::configure(const StreamConfiguration &inputCfg,
 		return -EINVAL;
 	}
 
+	inputBufferCount_ = inputCfg.bufferCount;
+	outputBufferCount_ = outputCfg.bufferCount;
+
 	return 0;
 }
 
@@ -204,13 +207,13 @@ int SimpleConverter::exportBuffers(unsigned int count,
 	return m2m_->capture()->exportBuffers(count, buffers);
 }
 
-int SimpleConverter::start(unsigned int count)
+int SimpleConverter::start()
 {
-	int ret = m2m_->output()->importBuffers(count);
+	int ret = m2m_->output()->importBuffers(inputBufferCount_);
 	if (ret < 0)
 		return ret;
 
-	ret = m2m_->capture()->importBuffers(count);
+	ret = m2m_->capture()->importBuffers(outputBufferCount_);
 	if (ret < 0) {
 		stop();
 		return ret;
diff --git a/src/libcamera/pipeline/simple/converter.h b/src/libcamera/pipeline/simple/converter.h
index 47a056e582d6..a1503a6099ff 100644
--- a/src/libcamera/pipeline/simple/converter.h
+++ b/src/libcamera/pipeline/simple/converter.h
@@ -43,7 +43,7 @@ public:
 	int exportBuffers(unsigned int count,
 			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-	int start(unsigned int count);
+	int start();
 	void stop();
 
 	int queueBuffers(FrameBuffer *input, FrameBuffer *output);
@@ -58,6 +58,9 @@ private:
 
 	std::queue<FrameBuffer *> captureDoneQueue_;
 	std::queue<FrameBuffer *> outputDoneQueue_;
+
+	unsigned int inputBufferCount_;
+	unsigned int outputBufferCount_;
 };
 
 } /* namespace libcamera */
diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index 1ed67bcec490..8c0bca36bbfb 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -607,6 +607,7 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c)
 		inputCfg.pixelFormat = pipeConfig.pixelFormat;
 		inputCfg.size = pipeConfig.captureSize;
 		inputCfg.stride = captureFormat.planes[0].bpl;
+		inputCfg.bufferCount = cfg.bufferCount;
 
 		ret = converter_->configure(inputCfg, cfg);
 		if (ret < 0) {
@@ -660,7 +661,7 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] ControlList *c
 	}
 
 	if (useConverter_) {
-		ret = converter_->start(count);
+		ret = converter_->start();
 		if (ret < 0) {
 			stop(camera);
 			return ret;