diff --git a/include/libcamera/internal/converter.h b/include/libcamera/internal/converter.h
index 834ec5ab..32490fe0 100644
--- a/include/libcamera/internal/converter.h
+++ b/include/libcamera/internal/converter.h
@@ -49,7 +49,8 @@ public:
 	virtual int exportBuffers(unsigned int output, unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers) = 0;
 
-	virtual int start() = 0;
+	virtual int start(unsigned int internalBufferCount,
+			  unsigned int bufferSlotCount) = 0;
 	virtual void stop() = 0;
 
 	virtual int queueBuffers(FrameBuffer *input,
diff --git a/include/libcamera/internal/converter/converter_v4l2_m2m.h b/include/libcamera/internal/converter/converter_v4l2_m2m.h
index 815916d0..1f471071 100644
--- a/include/libcamera/internal/converter/converter_v4l2_m2m.h
+++ b/include/libcamera/internal/converter/converter_v4l2_m2m.h
@@ -50,7 +50,8 @@ public:
 	int exportBuffers(unsigned int ouput, unsigned int count,
 			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-	int start();
+	int start(unsigned int internalBufferCount,
+		  unsigned int bufferSlotCount);
 	void stop();
 
 	int queueBuffers(FrameBuffer *input,
@@ -69,7 +70,8 @@ private:
 		int exportBuffers(unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-		int start();
+		int start(unsigned int internalBufferCount,
+			  unsigned int bufferSlotCount);
 		void stop();
 
 		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
diff --git a/src/libcamera/converter/converter_v4l2_m2m.cpp b/src/libcamera/converter/converter_v4l2_m2m.cpp
index 2a4d1d99..9d25f25a 100644
--- a/src/libcamera/converter/converter_v4l2_m2m.cpp
+++ b/src/libcamera/converter/converter_v4l2_m2m.cpp
@@ -107,13 +107,14 @@ int V4L2M2MConverter::Stream::exportBuffers(unsigned int count,
 	return m2m_->capture()->exportBuffers(count, buffers);
 }
 
-int V4L2M2MConverter::Stream::start()
+int V4L2M2MConverter::Stream::start(unsigned int internalBufferCount,
+				    unsigned int bufferSlotCount)
 {
-	int ret = m2m_->output()->importBuffers(inputBufferCount_);
+	int ret = m2m_->output()->importBuffers(internalBufferCount);
 	if (ret < 0)
 		return ret;
 
-	ret = m2m_->capture()->importBuffers(outputBufferCount_);
+	ret = m2m_->capture()->importBuffers(bufferSlotCount);
 	if (ret < 0) {
 		stop();
 		return ret;
@@ -373,12 +374,13 @@ int V4L2M2MConverter::exportBuffers(unsigned int output, unsigned int count,
 /**
  * \copydoc libcamera::Converter::start
  */
-int V4L2M2MConverter::start()
+int V4L2M2MConverter::start(unsigned int internalBufferCount,
+			    unsigned int bufferSlotCount)
 {
 	int ret;
 
 	for (Stream &stream : streams_) {
-		ret = stream.start();
+		ret = stream.start(internalBufferCount, bufferSlotCount);
 		if (ret < 0) {
 			stop();
 			return ret;
diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index 6b7c6d5c..196e5252 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -339,6 +339,7 @@ protected:
 
 private:
 	static constexpr unsigned int kNumInternalBuffers = 3;
+	static constexpr unsigned int kSimpleBufferSlotCount = 16;
 
 	struct EntityData {
 		std::unique_ptr<V4L2VideoDevice> video;
@@ -1232,17 +1233,27 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 		return -EBUSY;
 	}
 
+	/*
+	 * Number of internal buffers that will be used to move video capture
+	 * device frames to the converter.
+	 *
+	 * \todo Make this depend on the driver in use instead of being
+	 * hardcoded. In order to not drop frames, the realtime requirements for
+	 * each device should be checked, so it may not be as simple as just
+	 * using the minimum number of buffers required by the driver.
+	 */
+	static constexpr unsigned int internalBufferCount = 3;
+
 	if (data->useConverter_) {
 		/*
 		 * When using the converter allocate a fixed number of internal
 		 * buffers.
 		 */
-		ret = video->allocateBuffers(kNumInternalBuffers,
+		ret = video->allocateBuffers(internalBufferCount,
 					     &data->converterBuffers_);
 	} else {
-		/* Otherwise, prepare for using buffers from the only stream. */
-		Stream *stream = &data->streams_[0];
-		ret = video->importBuffers(stream->configuration().bufferCount);
+		/* Otherwise, prepare for using buffers. */
+		ret = video->importBuffers(kSimpleBufferSlotCount);
 	}
 	if (ret < 0) {
 		releasePipeline(data);
@@ -1258,7 +1269,8 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 	}
 
 	if (data->useConverter_) {
-		ret = data->converter_->start();
+		ret = data->converter_->start(internalBufferCount,
+					      kSimpleBufferSlotCount);
 		if (ret < 0) {
 			stop(camera);
 			return ret;