diff --git a/src/libcamera/framebuffer_allocator.cpp b/src/libcamera/framebuffer_allocator.cpp
index fc19a33a..dbd0db19 100644
--- a/src/libcamera/framebuffer_allocator.cpp
+++ b/src/libcamera/framebuffer_allocator.cpp
@@ -97,6 +97,10 @@ int FrameBufferAllocator::allocate(Stream *stream)
 		LOG(Allocator, Error)
 			<< "Stream is not part of " << camera_->id()
 			<< " active configuration";
+
+	if (ret < 0)
+		buffers_.erase(it);
+
 	return ret;
 }