diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index 35c29ceca..3b59087ae 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -1216,6 +1216,9 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate()
 	 * Then pick, among the possible configuration for the pixel format,
 	 * the smallest sensor resolution that can accommodate all streams
 	 * without upscaling.
+	 *
+	 * If both packed and unpacked Bayer capture formats are support prefer
+	 * packed for lower memory usage and better software ISP performance.
 	 */
 	const SimpleCameraData::Configuration *maxPipeConfig = nullptr;
 	pipeConfig_ = nullptr;
@@ -1223,12 +1226,25 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate()
 	for (const SimpleCameraData::Configuration *pipeConfig : *configs) {
 		const Size &captureSize = pipeConfig->captureSize;
 		const Size &maxOutputSize = pipeConfig->outputSizes.max;
+		bool curCfgPacked = false;
+		bool newCfgPacked = false;
+
+		if (pipeConfig_) {
+			BayerFormat curBayerFmt = BayerFormat::fromPixelFormat(pipeConfig_->captureFormat);
+			BayerFormat newBayerFmt = BayerFormat::fromPixelFormat(pipeConfig->captureFormat);
+			/* Only compare packed vs unpacked if both formats are Bayer */
+			if (curBayerFmt.isValid() && newBayerFmt.isValid()) {
+				curCfgPacked = curBayerFmt.packing == BayerFormat::Packing::CSI2;
+				newCfgPacked = newBayerFmt.packing == BayerFormat::Packing::CSI2;
+			}
+		}
 
 		if (maxOutputSize.width >= maxProcessedStreamSize.width &&
 		    maxOutputSize.height >= maxProcessedStreamSize.height &&
 		    captureSize.width >= maxRawStreamSize.width &&
 		    captureSize.height >= maxRawStreamSize.height) {
-			if (!pipeConfig_ || captureSize < pipeConfig_->captureSize)
+			if (!pipeConfig_ || captureSize < pipeConfig_->captureSize ||
+			    (captureSize <= pipeConfig_->captureSize && !curCfgPacked && newCfgPacked))
 				pipeConfig_ = pipeConfig;
 		}