diff --git a/include/libcamera/internal/converter.h b/include/libcamera/internal/converter.h
index 644ec429..1da51d1e 100644
--- a/include/libcamera/internal/converter.h
+++ b/include/libcamera/internal/converter.h
@@ -75,7 +75,7 @@ public:
 	virtual int exportBuffers(const Stream *stream, unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers) = 0;
 
-	virtual int start() = 0;
+	virtual int start(unsigned int inputBufferCount) = 0;
 	virtual void stop() = 0;
 
 	virtual int queueBuffers(FrameBuffer *input,
diff --git a/include/libcamera/internal/converter/converter_v4l2_m2m.h b/include/libcamera/internal/converter/converter_v4l2_m2m.h
index 0ad7bf7f..24997c6f 100644
--- a/include/libcamera/internal/converter/converter_v4l2_m2m.h
+++ b/include/libcamera/internal/converter/converter_v4l2_m2m.h
@@ -59,7 +59,7 @@ public:
 	int exportBuffers(const Stream *stream, unsigned int count,
 			  std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
 
-	int start() override;
+	int start(unsigned int inputBufferCount) override;
 	void stop() override;
 
 	int validateOutput(StreamConfiguration *cfg, bool *adjusted,
@@ -85,7 +85,7 @@ private:
 		int exportBuffers(unsigned int count,
 				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
 
-		int start();
+		int start(unsigned int inputBufferCount);
 		void stop();
 
 		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
diff --git a/src/libcamera/converter.cpp b/src/libcamera/converter.cpp
index d551b908..2c6f1e4b 100644
--- a/src/libcamera/converter.cpp
+++ b/src/libcamera/converter.cpp
@@ -190,6 +190,8 @@ Converter::~Converter()
 /**
  * \fn Converter::start()
  * \brief Start the converter streaming operation
+ * \param[in] inputBufferCount Number of input buffers that will be used to
+ * move video capture device frames into the converter.
  * \return 0 on success or a negative error code otherwise
  */
 
diff --git a/src/libcamera/converter/converter_v4l2_m2m.cpp b/src/libcamera/converter/converter_v4l2_m2m.cpp
index ee05b798..242e0d85 100644
--- a/src/libcamera/converter/converter_v4l2_m2m.cpp
+++ b/src/libcamera/converter/converter_v4l2_m2m.cpp
@@ -160,13 +160,15 @@ int V4L2M2MConverter::V4L2M2MStream::exportBuffers(unsigned int count,
 	return m2m_->capture()->exportBuffers(count, buffers);
 }
 
-int V4L2M2MConverter::V4L2M2MStream::start()
+int V4L2M2MConverter::V4L2M2MStream::start(unsigned int inputBufferCount)
 {
-	int ret = m2m_->output()->importBuffers(inputBufferCount_);
+	static constexpr unsigned int kOutputBufferCount = 16;
+
+	int ret = m2m_->output()->importBuffers(inputBufferCount);
 	if (ret < 0)
 		return ret;
 
-	ret = m2m_->capture()->importBuffers(outputBufferCount_);
+	ret = m2m_->capture()->importBuffers(kOutputBufferCount);
 	if (ret < 0) {
 		stop();
 		return ret;
@@ -620,12 +622,12 @@ V4L2M2MConverter::inputCropBounds(const Stream *stream)
 /**
  * \copydoc libcamera::Converter::start
  */
-int V4L2M2MConverter::start()
+int V4L2M2MConverter::start(unsigned int inputBufferCount)
 {
 	int ret;
 
 	for (auto &iter : streams_) {
-		ret = iter.second->start();
+		ret = iter.second->start(inputBufferCount);
 		if (ret < 0) {
 			stop();
 			return ret;
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index 81bf1c55..623bcfe5 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -1110,7 +1110,7 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL
 		actions += [&]() { stat_->streamOff(); };
 
 		if (useDewarper_) {
-			ret = dewarper_->start();
+			ret = dewarper_->start(kRkISP1InternalBufferCount);
 			if (ret) {
 				LOG(RkISP1, Error) << "Failed to start dewarper";
 				return ret;
diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index 1e77ccdc..4f4cba06 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -422,6 +422,7 @@ protected:
 
 private:
 	static constexpr unsigned int kNumInternalBuffers = 3;
+	static constexpr unsigned int kSimpleBufferSlotCount = 16;
 
 	struct EntityData {
 		std::unique_ptr<V4L2VideoDevice> video;
@@ -1466,17 +1467,27 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 		return -EBUSY;
 	}
 
+	/*
+	 * Number of internal buffers that will be used to move video capture
+	 * device frames to the converter.
+	 *
+	 * \todo Make this depend on the driver in use instead of being
+	 * hardcoded. In order to not drop frames, the realtime requirements for
+	 * each device should be checked, so it may not be as simple as just
+	 * using the minimum number of buffers required by the driver.
+	 */
+	static constexpr unsigned int internalBufferCount = 3;
+
 	if (data->useConversion_) {
 		/*
 		 * When using the converter allocate a fixed number of internal
 		 * buffers.
 		 */
-		ret = video->allocateBuffers(kNumInternalBuffers,
+		ret = video->allocateBuffers(internalBufferCount,
 					     &data->conversionBuffers_);
 	} else {
-		/* Otherwise, prepare for using buffers from the only stream. */
-		Stream *stream = &data->streams_[0];
-		ret = video->importBuffers(stream->configuration().bufferCount);
+		/* Otherwise, prepare for using buffers. */
+		ret = video->importBuffers(kSimpleBufferSlotCount);
 	}
 	if (ret < 0) {
 		releasePipeline(data);
@@ -1504,7 +1515,7 @@ int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlL
 
 	if (data->useConversion_) {
 		if (data->converter_)
-			ret = data->converter_->start();
+			ret = data->converter_->start(internalBufferCount);
 		else if (data->swIsp_)
 			ret = data->swIsp_->start();
 		else