diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp
index 651e0c93a7ab..8f75c5186b13 100644
--- a/src/libcamera/pipeline/ipu3/ipu3.cpp
+++ b/src/libcamera/pipeline/ipu3/ipu3.cpp
@@ -67,8 +67,6 @@ public:
 	const std::vector<const Stream *> &streams() { return streams_; }
 
 private:
-	void assignStreams();
-
 	/*
 	 * The IPU3CameraData instance is guaranteed to be valid as long as the
 	 * corresponding Camera instance is valid. In order to borrow a
@@ -133,50 +131,6 @@ IPU3CameraConfiguration::IPU3CameraConfiguration(Camera *camera,
 	data_ = data;
 }
 
-void IPU3CameraConfiguration::assignStreams()
-{
-	/*
-	 * Verify and update all configuration entries, and assign a stream to
-	 * each of them. The viewfinder stream can scale, while the output
-	 * stream can crop only, so select the output stream when the requested
-	 * resolution is equal to the sensor resolution, and the viewfinder
-	 * stream otherwise.
-	 */
-	std::set<const Stream *> availableStreams = {
-		&data_->outStream_,
-		&data_->vfStream_,
-		&data_->rawStream_,
-	};
-
-	/*
-	 * The caller is responsible to limit the number of requested streams
-	 * to a number supported by the pipeline before calling this function.
-	 */
-	ASSERT(availableStreams.size() >= config_.size());
-
-	streams_.clear();
-	streams_.reserve(config_.size());
-
-	for (const StreamConfiguration &cfg : config_) {
-		const PixelFormatInfo &info =
-			PixelFormatInfo::info(cfg.pixelFormat);
-		const Stream *stream;
-
-		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW)
-			stream = &data_->rawStream_;
-		else if (cfg.size == cio2Configuration_.size)
-			stream = &data_->outStream_;
-		else
-			stream = &data_->vfStream_;
-
-		if (availableStreams.find(stream) == availableStreams.end())
-			stream = *availableStreams.begin();
-
-		streams_.push_back(stream);
-		availableStreams.erase(stream);
-	}
-}
-
 CameraConfiguration::Status IPU3CameraConfiguration::validate()
 {
 	Status status = Valid;
@@ -209,9 +163,6 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()
 	if (!cio2Configuration_.pixelFormat.isValid())
 		return Invalid;
 
-	/* Assign streams to each configuration entry. */
-	assignStreams();
-
 	/* Verify and adjust configuration if needed. */
 	unsigned int rawCount = 0;
 	unsigned int outCount = 0;
@@ -385,9 +336,6 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)
 	IPU3CameraConfiguration *config =
 		static_cast<IPU3CameraConfiguration *>(c);
 	IPU3CameraData *data = cameraData(camera);
-	Stream *outStream = &data->outStream_;
-	Stream *vfStream = &data->vfStream_;
-	CIO2Device *cio2 = &data->cio2_;
 	ImgUDevice *imgu = data->imgu_;
 	V4L2DeviceFormat outputFormat;
 	int ret;
@@ -435,12 +383,36 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)
 		return ret;
 
 	/*
-	 * Pass the requested stream size to the CIO2 unit and get back the
-	 * adjusted format to be propagated to the ImgU output devices.
+	 * Identify the maximum sizes and the number of non-raw streams
+	 * requested.
+	 */
+	unsigned int outCount = 0;
+	Size maxOut;
+	for (unsigned int i = 0; i < config->size(); ++i) {
+		const StreamConfiguration &cfg = (*config)[i];
+		const PixelFormatInfo &info =
+			PixelFormatInfo::info(cfg.pixelFormat);
+
+		/*
+		 * We have validated that only one raw stream can be requested
+		 * and the CIO2 configuration has been set during validation.
+		 */
+		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW)
+			continue;
+
+		/* Get the largest size of the two available processed streams. */
+		if (maxOut < cfg.size)
+			maxOut = cfg.size;
+		outCount++;
+	}
+
+	/*
+	 * Configure the CIO2 unit with the format computed during validation
+	 * and apply the same format to the ImgU input.
 	 */
 	const Size &sensorSize = config->cio2Format().size;
 	V4L2DeviceFormat cio2Format = {};
-	ret = cio2->configure(sensorSize, &cio2Format);
+	ret = data->cio2_.configure(sensorSize, &cio2Format);
 	if (ret)
 		return ret;
 
@@ -451,40 +423,60 @@ int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)
 	/* Apply the format to the configured streams output devices. */
 	bool outActive = false;
 	bool vfActive = false;
-
 	for (unsigned int i = 0; i < config->size(); ++i) {
-		/*
-		 * Use a const_cast<> here instead of storing a mutable stream
-		 * pointer in the configuration to let the compiler catch
-		 * unwanted modifications of camera data in the configuration
-		 * validate() implementation.
-		 */
-		Stream *stream = const_cast<Stream *>(config->streams()[i]);
 		StreamConfiguration &cfg = (*config)[i];
+		const PixelFormatInfo &info =
+			PixelFormatInfo::info(cfg.pixelFormat);
+		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW) {
+			/*
+			 * The RAW stream is configured as part of the CIO2 and
+			 * no configuration is needed for the ImgU.
+			 */
+			cfg.setStream(&data->rawStream_);
+			cfg.stride = cio2Format.planes[0].bpl;
 
-		cfg.setStream(stream);
+			LOG(IPU3, Debug) << "Assigned " << cfg.toString()
+					 << " to the raw output";
+			continue;
+		}
 
-		if (stream == outStream) {
+		/*
+		 * The viewfinder stream can scale, while the main output stream
+		 * can crop only. Select the main output for the largest
+		 * non-raw stream only if another smaller one has been
+		 * requested.
+		 */
+		if (cfg.size == maxOut && outCount > 1) {
 			ret = imgu->configureOutput(cfg, &outputFormat);
 			if (ret)
 				return ret;
 
 			cfg.stride = outputFormat.planes[0].bpl;
+			cfg.setStream(&data->outStream_);
 			outActive = true;
-		} else if (stream == vfStream) {
-			ret = imgu->configureViewfinder(cfg, &outputFormat);
-			if (ret)
-				return ret;
 
-			cfg.stride = outputFormat.planes[0].bpl;
-			vfActive = true;
-		} else {
-			/*
-			 * The RAW stream is configured as part of the CIO2 and
-			 * no configuration is needed for the ImgU.
-			 */
-			cfg.stride = cio2Format.planes[0].bpl;
+			LOG(IPU3, Debug) << "Assigned " << cfg.toString()
+					 << " to the main output";
+			continue;
 		}
+
+		/*
+		 * If a single non-raw stream has been requested, assign it to
+		 * the viewfinder output regardless of its size. If more than
+		 * one non-raw stream has been requested, assign to viewfinder
+		 * the smaller one to reduce the cropping required (if any) on
+		 * the main output.
+		 */
+		ret = imgu->configureViewfinder(cfg, &outputFormat);
+		if (ret)
+			return ret;
+
+		cfg.stride = outputFormat.planes[0].bpl;
+		cfg.setStream(&data->vfStream_);
+		vfActive = true;
+
+		LOG(IPU3, Debug) << "Assigned " << cfg.toString()
+				 << " to the viewfinder output";
 	}
 
 	/*
