diff --git a/src/libcamera/include/v4l2_videodevice.h b/src/libcamera/include/v4l2_videodevice.h
index fcf072641420dacf..f04feed87b24f28f 100644
--- a/src/libcamera/include/v4l2_videodevice.h
+++ b/src/libcamera/include/v4l2_videodevice.h
@@ -125,6 +125,7 @@ private:
 		bool operator==(const FrameBuffer &buffer);
 
 		bool free;
+		utils::time_point lastUsed;
 
 	private:
 		struct Plane {
diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 99470ce11421c77c..9a4d2479b20f5e27 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -205,6 +205,7 @@ int V4L2BufferCache::get(const FrameBuffer &buffer)
 {
 	bool hit = false;
 	int use = -1;
+	utils::time_point oldest = utils::clock::now();
 
 	for (unsigned int index = 0; index < cache_.size(); index++) {
 		const Entry &entry = cache_[index];
@@ -212,8 +213,10 @@ int V4L2BufferCache::get(const FrameBuffer &buffer)
 		if (!entry.free)
 			continue;
 
-		if (use < 0)
+		if (cache_[index].lastUsed < oldest) {
 			use = index;
+			oldest = cache_[index].lastUsed;
+		}
 
 		/* Try to find a cache hit by comparing the planes. */
 		if (cache_[index] == buffer) {
@@ -245,12 +248,12 @@ void V4L2BufferCache::put(unsigned int index)
 }
 
 V4L2BufferCache::Entry::Entry()
-	: free(true)
+	: free(true), lastUsed(utils::clock::now())
 {
 }
 
 V4L2BufferCache::Entry::Entry(bool free, const FrameBuffer &buffer)
-	: free(free)
+	: free(free), lastUsed(utils::clock::now())
 {
 	for (const FrameBuffer::Plane &plane : buffer.planes())
 		planes_.emplace_back(plane);