diff --git a/src/libcamera/include/v4l2_videodevice.h b/src/libcamera/include/v4l2_videodevice.h
index e4d35ab..a77b62e 100644
--- a/src/libcamera/include/v4l2_videodevice.h
+++ b/src/libcamera/include/v4l2_videodevice.h
@@ -125,6 +125,7 @@ private:
 		bool operator==(const FrameBuffer &buffer);
 
 		bool free;
+		bool occupied;
 
 	private:
 		struct Plane {
diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 37b8d33..b887a38 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -212,7 +212,7 @@ int V4L2BufferCache::get(const FrameBuffer &buffer)
 		if (!entry.free)
 			continue;
 
-		if (use < 0)
+		if (use < 0 && !entry.occupied)
 			use = index;
 
 		/* Try to find a cache hit by comparing the planes. */
@@ -245,12 +245,12 @@ void V4L2BufferCache::put(unsigned int index)
 }
 
 V4L2BufferCache::Entry::Entry()
-	: free(true)
+	: free(true), occupied(false)
 {
 }
 
 V4L2BufferCache::Entry::Entry(bool free, const FrameBuffer &buffer)
-	: free(free)
+	: free(free), occupied(true)
 {
 	for (const FrameBuffer::Plane &plane : buffer.planes())
 		planes_.emplace_back(plane);