diff --git a/src/ipa/mali-c55/algorithms/agc.cpp b/src/ipa/mali-c55/algorithms/agc.cpp
index 014fd12452ac..2acc45cc0b04 100644
--- a/src/ipa/mali-c55/algorithms/agc.cpp
+++ b/src/ipa/mali-c55/algorithms/agc.cpp
@@ -137,6 +137,7 @@ int Agc::init(IPAContext &context, const YamlObject &tuningData)
 	if (ret)
 		return ret;
 
+	/* \todo Support AnalogueGainMode and ExposureTimeMode */
 	context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true);
 	context.ctrlMap[&controls::DigitalGain] = ControlInfo(
 		static_cast<float>(kMinDigitalGain),
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp
index 1ecaff680978..55cadb1793a5 100644
--- a/src/ipa/rkisp1/algorithms/agc.cpp
+++ b/src/ipa/rkisp1/algorithms/agc.cpp
@@ -155,8 +155,6 @@ int Agc::init(IPAContext &context, const YamlObject &tuningData)
 		ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto),
 				ControlValue(controls::AnalogueGainModeManual) } },
 			    ControlValue(controls::AnalogueGainModeAuto));
-	/* \todo Move this to the Camera class */
-	context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true);
 	context.ctrlMap[&controls::ExposureValue] = ControlInfo(-8.0f, 8.0f, 0.0f);
 	context.ctrlMap.merge(controls());
 
diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp
index 322694985e72..eceb9d14d74d 100644
--- a/src/ipa/rpi/common/ipa_base.cpp
+++ b/src/ipa/rpi/common/ipa_base.cpp
@@ -56,8 +56,6 @@ constexpr Duration controllerMinFrameDuration = 1.0s / 30.0;
 
 /* List of controls handled by the Raspberry Pi IPA */
 const ControlInfoMap::Map ipaControls{
-	/* \todo Move this to the Camera class */
-	{ &controls::AeEnable, ControlInfo(false, true, true) },
 	{ &controls::ExposureTimeMode,
 	  ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto),
 			  ControlValue(controls::ExposureTimeModeManual) } },
diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp
index 4ed9a21726d9..29042fce1466 100644
--- a/src/libcamera/camera.cpp
+++ b/src/libcamera/camera.cpp
@@ -1339,33 +1339,6 @@ std::unique_ptr<Request> Camera::createRequest(uint64_t cookie)
 	return request;
 }
 
-/**
- * \brief Patch a control list that contains the AeEnable control
- * \param[inout] controls The control list to be patched
- *
- * The control list is patched in place, turning the AeEnable control into
- * the equivalent ExposureTimeMode/AnalogueGainMode controls.
- */
-void Camera::patchControlList(ControlList &controls)
-{
-	const auto &aeEnable = controls.get(controls::AeEnable);
-	if (aeEnable) {
-		if (_d()->controlInfo_.count(controls::AnalogueGainMode.id()) &&
-		    !controls.contains(controls::AnalogueGainMode.id())) {
-			controls.set(controls::AnalogueGainMode,
-				     *aeEnable ? controls::AnalogueGainModeAuto
-					       : controls::AnalogueGainModeManual);
-		}
-
-		if (_d()->controlInfo_.count(controls::ExposureTimeMode.id()) &&
-		    !controls.contains(controls::ExposureTimeMode.id())) {
-			controls.set(controls::ExposureTimeMode,
-				     *aeEnable ? controls::ExposureTimeModeAuto
-					       : controls::ExposureTimeModeManual);
-		}
-	}
-}
-
 /**
  * \brief Queue a request to the camera
  * \param[in] request The request to queue to the camera
@@ -1434,9 +1407,6 @@ int Camera::queueRequest(Request *request)
 		}
 	}
 
-	/* Pre-process AeEnable. */
-	patchControlList(request->controls());
-
 	d->layers_->queueRequest(request);
 
 	d->pipe_->invokeMethod(&PipelineHandler::queueRequest,
@@ -1477,16 +1447,8 @@ int Camera::start(const ControlList *controls)
 
 	ControlList *newControls = d->layers_->start(controls);
 
-	if (controls) {
-		ControlList copy(*newControls);
-		patchControlList(copy);
-		ret = d->pipe_->invokeMethod(&PipelineHandler::start,
-					     ConnectionTypeBlocking, this, &copy);
-	} else {
-		ret = d->pipe_->invokeMethod(&PipelineHandler::start,
-					     ConnectionTypeBlocking, this, nullptr);
-	}
-
+	ret = d->pipe_->invokeMethod(&PipelineHandler::start,
+				     ConnectionTypeBlocking, this, newControls);
 	if (ret)
 		return ret;
 
diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp
index 3bd51733d400..bad20d2c7712 100644
--- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp
+++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp
@@ -610,11 +610,6 @@ int UVCCameraData::init(std::shared_ptr<MediaDevice> media)
 		addControl(cid, info, &ctrls);
 	}
 
-	if (autoExposureMode_ && manualExposureMode_) {
-		/* \todo Move this to the Camera class */
-		ctrls[&controls::AeEnable] = ControlInfo(false, true, true);
-	}
-
 	controlInfo_ = ControlInfoMap(std::move(ctrls), controls::controls);
 
 	/*