diff --git a/src/libcamera/include/v4l2_videodevice.h b/src/libcamera/include/v4l2_videodevice.h
index 34bbff41760753bd..254f8797af42dd8a 100644
--- a/src/libcamera/include/v4l2_videodevice.h
+++ b/src/libcamera/include/v4l2_videodevice.h
@@ -12,6 +12,7 @@
 
 #include <linux/videodev2.h>
 
+#include <libcamera/buffer.h>
 #include <libcamera/geometry.h>
 #include <libcamera/pixelformats.h>
 #include <libcamera/signal.h>
@@ -22,7 +23,6 @@
 
 namespace libcamera {
 
-class Buffer;
 class BufferMemory;
 class BufferPool;
 class EventNotifier;
@@ -105,6 +105,24 @@ struct V4L2Capability final : v4l2_capability {
 	}
 };
 
+class V4L2BufferCache
+{
+public:
+	V4L2BufferCache(unsigned int size);
+	V4L2BufferCache(const std::vector<FrameBuffer *> buffers);
+
+	int fetch(const FrameBuffer *buffer);
+	void put(unsigned int index);
+
+private:
+	struct CacheInfo {
+		bool free;
+		std::vector<FrameBuffer::Plane> last;
+	};
+
+	std::vector<CacheInfo> cache_;
+};
+
 class V4L2DeviceFormat
 {
 public:
diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index a05dd6a1f7d86eaa..c82f2829601bd14c 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -19,7 +19,6 @@
 
 #include <linux/drm_fourcc.h>
 
-#include <libcamera/buffer.h>
 #include <libcamera/event_notifier.h>
 
 #include "log.h"
@@ -134,6 +133,110 @@ LOG_DECLARE_CATEGORY(V4L2)
  * \return True if the video device provides Streaming I/O IOCTLs
  */
 
+/**
+ * \class V4L2BufferCache
+ * \brief Hot cache of associations between V4L2 index and FrameBuffer
+ *
+ * There is performance to be gained if the same V4L2 buffer index can be
+ * reused for the same FrameBuffer object as the kernel don't have to redo
+ * the mapping. The V4L2BufferCache tries to keep a hot-cache of mappings
+ * between the two.
+ *
+ * If there is a cache miss is not critical, everything still works as expected.
+ */
+
+/**
+ * \brief Create a empty cache of a given size
+ * \param[in] size Size of cache to create
+ *
+ * Create a cold cache with \a size entries. The cache will be populated as
+ * it's being used.
+ */
+V4L2BufferCache::V4L2BufferCache(unsigned int size)
+{
+	cache_.resize(size, { .free = true, .last = {} });
+}
+
+/**
+ * \brief Create a pre-populated cache
+ * \param[in] buffers Array of buffers to pre-populated with
+ *
+ * Create a warm cache from \a buffers.
+ */
+V4L2BufferCache::V4L2BufferCache(const std::vector<FrameBuffer *> buffers)
+{
+	for (const FrameBuffer *buffer : buffers)
+		cache_.push_back({ .free = true, .last = buffer->planes() });
+}
+
+/**
+ * \brief Fetch a index from the cache
+ * \param[in] buffer FrameBuffer to match
+ *
+ * Try to find \a buffer in cache and if it's free reuse the last used index
+ * for this buffer. If the buffer have never been seen or if have been evinced
+ * from the cache the first free index is pieced instead. Likewise if the last
+ * used index is in use a new free index is picked.
+ *
+ * When an index is picked it is marked as in-use and returned to the caller.
+ * The association is also recorded so it if possible can reused the next time
+ * the FrameBuffer is seen.
+ *
+ * \return V4L2 buffer index
+ */
+int V4L2BufferCache::fetch(const FrameBuffer *buffer)
+{
+	int use = -1;
+
+	for (unsigned int index = 0; index < cache_.size(); index++) {
+		if (!cache_[index].free)
+			continue;
+
+		if (use < 0)
+			use = index;
+
+		/* Try to find a cache hit by comparing the planes. */
+		std::vector<FrameBuffer::Plane> planes = buffer->planes();
+		if (cache_[index].last.size() != planes.size())
+			continue;
+
+		bool match = true;
+		for (unsigned int i = 0; i < planes.size(); i++) {
+			if (cache_[index].last[i].fd != planes[i].fd ||
+			    cache_[index].last[i].length != planes[i].length) {
+				match = false;
+				break;
+			}
+		}
+
+		if (!match)
+			continue;
+
+		use = index;
+		break;
+	}
+
+	if (use < 0)
+		return -ENOENT;
+
+	cache_[use].free = false;
+	cache_[use].last = buffer->planes();
+
+	return use;
+}
+
+/**
+ * \brief Pit a V4L2 index back in the cache
+ * \param[in] index V4L2 index
+ *
+ * Mark the \a index as free in the cache so it can be reused.
+ */
+void V4L2BufferCache::put(unsigned int index)
+{
+	ASSERT(index < cache_.size());
+	cache_[index].free = true;
+}
+
 /**
  * \class V4L2DeviceFormat
  * \brief The V4L2 video device image format and sizes