diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp
index 23f4c14d..5ea57399 100644
--- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp
+++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp
@@ -172,11 +172,14 @@ pisp_image_format_config toPiSPImageFormat(V4L2DeviceFormat &format)
 		image.stride2 = image.stride;
 		break;
 	case formats::NV21:
+		/*
+		 * ORDER_SWAPPED does not work with semi-planar formats, so
+		 * we going to have swap rows in the output CSC matrix.
+		 */
 		image.format = PISP_IMAGE_FORMAT_THREE_CHANNEL +
 			       PISP_IMAGE_FORMAT_BPS_8 +
 			       PISP_IMAGE_FORMAT_SAMPLING_420 +
-			       PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR +
-			       PISP_IMAGE_FORMAT_ORDER_SWAPPED;
+			       PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR;
 		image.stride2 = image.stride;
 		break;
 	case formats::YUYV:
@@ -200,11 +203,14 @@ pisp_image_format_config toPiSPImageFormat(V4L2DeviceFormat &format)
 		image.stride2 = image.stride;
 		break;
 	case formats::NV61:
+		/*
+		 * ORDER_SWAPPED does not work with semi-planar formats, so
+		 * we going to have swap rows in the output CSC matrix.
+		 */
 		image.format = PISP_IMAGE_FORMAT_THREE_CHANNEL +
 			       PISP_IMAGE_FORMAT_BPS_8 +
 			       PISP_IMAGE_FORMAT_SAMPLING_422 +
-			       PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR +
-			       PISP_IMAGE_FORMAT_ORDER_SWAPPED;
+			       PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR;
 		image.stride2 = image.stride;
 		break;
 	case formats::RGB888:
@@ -1964,6 +1970,19 @@ bool PiSPCameraData::calculateCscConfiguration(const V4L2DeviceFormat &v4l2Forma
 				<< ", defaulting to sYCC";
 			be_->InitialiseYcbcr(csc, "jpeg");
 		}
+
+		if (pixFormat == formats::NV21 || pixFormat == formats::NV61) {
+			/*
+			 * The ORDER_SWAPPED flag doesn't work with semi-planar formats,
+			 * so instead we have to swap 2 matrix rows.
+			 */
+			pisp_be_ccm_config copy = csc;
+			memcpy(&csc.coeffs[3], &copy.coeffs[6], 3 * sizeof(csc.coeffs[0]));
+			memcpy(&csc.coeffs[6], &copy.coeffs[3], 3 * sizeof(csc.coeffs[0]));
+			csc.offsets[1] = copy.offsets[2];
+			csc.offsets[2] = copy.offsets[1];
+		}
+
 		return true;
 	}
 	/* There will be more formats to check for in due course. */