diff --git a/src/libcamera/framebuffer.cpp b/src/libcamera/framebuffer.cpp
index e71c2ffae034..e4f8419a9063 100644
--- a/src/libcamera/framebuffer.cpp
+++ b/src/libcamera/framebuffer.cpp
@@ -210,6 +210,8 @@ FrameBuffer::FrameBuffer(const std::vector<Plane> &planes, unsigned int cookie)
 	: Extensible(std::make_unique<Private>()), planes_(planes),
 	  cookie_(cookie)
 {
+	metadata_.planes.resize(planes_.size());
+
 	unsigned int offset = 0;
 	bool isContiguous = true;
 	ino_t inode = 0;
diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 281f3f13fc9c..81209e485c24 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -1675,7 +1675,6 @@ FrameBuffer *V4L2VideoDevice::dequeueBuffer()
 
 	unsigned int numV4l2Planes = multiPlanar ? buf.length : 1;
 	FrameMetadata &metadata = buffer->metadata_;
-	metadata.planes.clear();
 
 	if (numV4l2Planes != buffer->planes().size()) {
 		/*
@@ -1705,8 +1704,9 @@ FrameBuffer *V4L2VideoDevice::dequeueBuffer()
 				return buffer;
 			}
 
-			metadata.planes.push_back({ std::min(plane.length, bytesused) });
-			bytesused -= metadata.planes.back().bytesused;
+			metadata.planes[i].bytesused =
+				std::min(plane.length, bytesused);
+			bytesused -= metadata.planes[i].bytesused;
 		}
 	} else if (multiPlanar) {
 		/*
@@ -1715,9 +1715,9 @@ FrameBuffer *V4L2VideoDevice::dequeueBuffer()
 		 * V4L2 buffer is guaranteed to be equal at this point.
 		 */
 		for (unsigned int i = 0; i < numV4l2Planes; ++i)
-			metadata.planes.push_back({ planes[i].bytesused });
+			metadata.planes[i].bytesused = planes[i].bytesused;
 	} else {
-		metadata.planes.push_back({ buf.bytesused });
+		metadata.planes[0].bytesused = buf.bytesused;
 	}
 
 	return buffer;