diff --git a/include/libcamera/internal/converter.h b/include/libcamera/internal/converter.h
index 834ec5ab..e8a5aeca 100644
--- a/include/libcamera/internal/converter.h
+++ b/include/libcamera/internal/converter.h
@@ -49,7 +49,7 @@ public:
 	virtual int exportBuffers(unsigned int output, unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers) = 0;
 
-	virtual int start() = 0;
+	virtual int start(unsigned int inputBufferCount) = 0;
 	virtual void stop() = 0;
 
 	virtual int queueBuffers(FrameBuffer *input,
diff --git a/include/libcamera/internal/converter/converter_v4l2_m2m.h b/include/libcamera/internal/converter/converter_v4l2_m2m.h
index 815916d0..8aedb6df 100644
--- a/include/libcamera/internal/converter/converter_v4l2_m2m.h
+++ b/include/libcamera/internal/converter/converter_v4l2_m2m.h
@@ -50,7 +50,7 @@ public:
 	int exportBuffers(unsigned int ouput, unsigned int count,
 			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-	int start();
+	int start(unsigned int inputBufferCount);
 	void stop();
 
 	int queueBuffers(FrameBuffer *input,
@@ -69,7 +69,7 @@ private:
 		int exportBuffers(unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-		int start();
+		int start(unsigned int inputBufferCount);
 		void stop();
 
 		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
diff --git a/src/libcamera/converter.cpp b/src/libcamera/converter.cpp
index 3de39cff..2413424b 100644
--- a/src/libcamera/converter.cpp
+++ b/src/libcamera/converter.cpp
@@ -127,6 +127,8 @@ Converter::~Converter()
 /**
  * \fn Converter::start()
  * \brief Start the converter streaming operation
+ * \param[in] inputBufferCount Number of input buffers that will be used to
+ * move video capture device frames into the converter.
  * \return 0 on success or a negative error code otherwise
  */
 
diff --git a/src/libcamera/converter/converter_v4l2_m2m.cpp b/src/libcamera/converter/converter_v4l2_m2m.cpp
index 2a4d1d99..261c3329 100644
--- a/src/libcamera/converter/converter_v4l2_m2m.cpp
+++ b/src/libcamera/converter/converter_v4l2_m2m.cpp
@@ -107,13 +107,15 @@ int V4L2M2MConverter::Stream::exportBuffers(unsigned int count,
 	return m2m_->capture()->exportBuffers(count, buffers);
 }
 
-int V4L2M2MConverter::Stream::start()
+int V4L2M2MConverter::Stream::start(unsigned int inputBufferCount)
 {
-	int ret = m2m_->output()->importBuffers(inputBufferCount_);
+	static constexpr unsigned int kOutputBufferCount = 16;
+
+	int ret = m2m_->output()->importBuffers(inputBufferCount);
 	if (ret < 0)
 		return ret;
 
-	ret = m2m_->capture()->importBuffers(outputBufferCount_);
+	ret = m2m_->capture()->importBuffers(kOutputBufferCount);
 	if (ret < 0) {
 		stop();
 		return ret;
@@ -373,12 +375,12 @@ int V4L2M2MConverter::exportBuffers(unsigned int output, unsigned int count,
 /**
  * \copydoc libcamera::Converter::start
  */
-int V4L2M2MConverter::start()
+int V4L2M2MConverter::start(unsigned int inputBufferCount)
 {
 	int ret;
 
 	for (Stream &stream : streams_) {
-		ret = stream.start();
+		ret = stream.start(inputBufferCount);
 		if (ret < 0) {
 			stop();
 			return ret;
diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index 6b7c6d5c..64f341e1 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -339,6 +339,7 @@ protected:
 
 private:
 	static constexpr unsigned int kNumInternalBuffers = 3;
+	static constexpr unsigned int kSimpleBufferSlotCount = 16;
 
 	struct EntityData {
 		std::unique_ptr<V4L2VideoDevice> video;
@@ -1232,17 +1233,27 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 		return -EBUSY;
 	}
 
+	/*
+	 * Number of internal buffers that will be used to move video capture
+	 * device frames to the converter.
+	 *
+	 * \todo Make this depend on the driver in use instead of being
+	 * hardcoded. In order to not drop frames, the realtime requirements for
+	 * each device should be checked, so it may not be as simple as just
+	 * using the minimum number of buffers required by the driver.
+	 */
+	static constexpr unsigned int internalBufferCount = 3;
+
 	if (data->useConverter_) {
 		/*
 		 * When using the converter allocate a fixed number of internal
 		 * buffers.
 		 */
-		ret = video->allocateBuffers(kNumInternalBuffers,
+		ret = video->allocateBuffers(internalBufferCount,
 					     &data->converterBuffers_);
 	} else {
-		/* Otherwise, prepare for using buffers from the only stream. */
-		Stream *stream = &data->streams_[0];
-		ret = video->importBuffers(stream->configuration().bufferCount);
+		/* Otherwise, prepare for using buffers. */
+		ret = video->importBuffers(kSimpleBufferSlotCount);
 	}
 	if (ret < 0) {
 		releasePipeline(data);
@@ -1258,7 +1269,7 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 	}
 
 	if (data->useConverter_) {
-		ret = data->converter_->start();
+		ret = data->converter_->start(internalBufferCount);
 		if (ret < 0) {
 			stop(camera);
 			return ret;