[v8,3/3] libcamera: android: Add face detection control support
diff mbox series

Message ID 20240925081551.3265942-4-chenghaoyang@google.com
State Accepted
Headers show
Series
  • Add Face Detection Controls
Related show

Commit Message

Harvey Yang Sept. 25, 2024, 8:12 a.m. UTC 
From: Harvey Yang <chenghaoyang@chromium.org>

Allow Android HAL adapter to pass the face detection metadata control to
the pipeline and also send face detection metadata to the camera client
if the pipeline generates it.

Signed-off-by: Yudhistira Erlandinata <yerlandinata@chromium.org>
Co-developed-by: Harvey Yang <chenghaoyang@chromium.org>
Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
---
 src/android/camera_capabilities.cpp | 45 +++++++++++++++++--
 src/android/camera_device.cpp       | 70 ++++++++++++++++++++++++++++-
 2 files changed, 109 insertions(+), 6 deletions(-)

Comments

Jacopo Mondi Sept. 30, 2024, 1:49 p.m. UTC | #1 
Hi Harvey

  While rebasing this patch I noticed something

On Wed, Sep 25, 2024 at 08:12:26AM GMT, Harvey Yang wrote:
> From: Harvey Yang <chenghaoyang@chromium.org>
>
> Allow Android HAL adapter to pass the face detection metadata control to
> the pipeline and also send face detection metadata to the camera client
> if the pipeline generates it.
>
> Signed-off-by: Yudhistira Erlandinata <yerlandinata@chromium.org>
> Co-developed-by: Harvey Yang <chenghaoyang@chromium.org>
> Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
> ---
>  src/android/camera_capabilities.cpp | 45 +++++++++++++++++--
>  src/android/camera_device.cpp       | 70 ++++++++++++++++++++++++++++-
>  2 files changed, 109 insertions(+), 6 deletions(-)
>
> diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
> index 71043e12..a89a3115 100644
> --- a/src/android/camera_capabilities.cpp
> +++ b/src/android/camera_capabilities.cpp
> @@ -7,6 +7,8 @@
>
>  #include "camera_capabilities.h"
>
> +#include <stdint.h>
> +
>  #include <algorithm>
>  #include <array>
>  #include <cmath>
> @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata()
>  				  maxFrameDuration_);
>
>  	/* Statistics static metadata. */
> -	uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> -	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> -				  faceDetectMode);
> -
>  	int32_t maxFaceCount = 0;
> +	auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
> +	if (iter != camera_->controls().end()) {
> +		const ControlInfo &faceDetectCtrlInfo = iter->second;
> +		std::vector<uint8_t> faceDetectModes;
> +		bool hasFaceDetection = false;
> +
> +		for (const auto &value : faceDetectCtrlInfo.values()) {
> +			uint8_t mode = value.get<uint8_t>();
> +			uint8_t androidMode = 0;
> +
> +			switch (mode) {
> +			case controls::draft::FaceDetectModeOff:
> +				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> +				break;
> +			case controls::draft::FaceDetectModeSimple:
> +				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
> +				hasFaceDetection = true;
> +				break;
> +			default:
> +				LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
> +			}
> +			faceDetectModes.push_back(androidMode);
> +		}
> +		if (hasFaceDetection) {
> +			/*
> +			 * \todo Create new libcamera controls to query max
> +			 * possible faces detected.
> +			 */
> +			maxFaceCount = 10;
> +			staticMetadata_->addEntry(
> +				ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> +				faceDetectModes.data(), faceDetectModes.size());
> +		}
> +	} else {
> +		uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> +		staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> +					  faceDetectMode);
> +	}
> +
>  	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
>  				  maxFaceCount);
>
> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
> index 493f66e7..e80b9e17 100644
> --- a/src/android/camera_device.cpp
> +++ b/src/android/camera_device.cpp
> @@ -15,6 +15,7 @@
>  #include <vector>
>
>  #include <libcamera/base/log.h>
> +#include <libcamera/base/span.h>
>  #include <libcamera/base/unique_fd.h>
>  #include <libcamera/base/utils.h>
>
> @@ -22,6 +23,7 @@
>  #include <libcamera/controls.h>
>  #include <libcamera/fence.h>
>  #include <libcamera/formats.h>
> +#include <libcamera/geometry.h>
>  #include <libcamera/property_ids.h>
>
>  #include "system/graphics.h"
> @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
>  		controls.set(controls::ScalerCrop, cropRegion);
>  	}
>
> +	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
> +		const uint8_t *data = entry.data.u8;
> +		controls.set(controls::draft::FaceDetectMode, data[0]);
> +	}
> +
>  	if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
>  		const int32_t data = *entry.data.i32;
>  		int32_t testPatternMode = controls::draft::TestPatternModeOff;
> @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
>  	value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
>  	resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
>
> -	value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> -	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
> +	settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry);
> +	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
> +				 entry.data.u8, 1);

Should this be unconditionally populated or should we rather

	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
         	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
				 entry.data.u8, 1);
        }

like it is done for other metadata populated directly with the value
received from controls ?

in example:
	if (settings.getEntry(ANDROID_LENS_APERTURE, &entry))
		resultMetadata->addEntry(ANDROID_LENS_APERTURE, entry.data.f, 1);

a few lines above

>
>  	value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
>  	resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
> @@ -1580,6 +1588,64 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
>  		resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
>  					 *frameDuration * 1000);
>
> +	const auto &faceDetectRectangles =
> +		metadata.get(controls::draft::FaceDetectFaceRectangles);
> +	if (faceDetectRectangles) {
> +		std::vector<int32_t> flatRectangles;
> +		for (const Rectangle &rect : *faceDetectRectangles) {
> +			flatRectangles.push_back(rect.x);
> +			flatRectangles.push_back(rect.y);
> +			flatRectangles.push_back(rect.x + rect.width);
> +			flatRectangles.push_back(rect.y + rect.height);
> +		}
> +		resultMetadata->addEntry(
> +			ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
> +	}
> +
> +	const auto &faceDetectFaceScores =
> +		metadata.get(controls::draft::FaceDetectFaceScores);
> +	if (faceDetectRectangles && faceDetectFaceScores) {
> +		if (faceDetectFaceScores->size() != faceDetectRectangles->size()) {
> +			LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
> +					<< "Expected: "
> +					<< faceDetectRectangles->size()
> +					<< ", got: " << faceDetectFaceScores->size();
> +		}
> +		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES,
> +					 *faceDetectRectangles);

s/faceDetectRectangles/faceDetectFaceScores

This seems a bug: can you see scores in the application you're testing
with ?

With your ack, I'll fix the above in the series I'm about to send.

Thanks
  j


> +	}
> +
> +	const auto &faceDetectFaceLandmarks =
> +		metadata.get(controls::draft::FaceDetectFaceLandmarks);
> +	if (faceDetectRectangles && faceDetectFaceLandmarks) {
> +		size_t expectedLandmarks = faceDetectRectangles->size() * 3;
> +		if (faceDetectFaceLandmarks->size() != expectedLandmarks) {
> +			LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
> +					<< "Expected: " << expectedLandmarks
> +					<< ", got: " << faceDetectFaceLandmarks->size();
> +		}
> +
> +		std::vector<int32_t> androidLandmarks;
> +		for (const auto &landmark : *faceDetectFaceLandmarks) {
> +			androidLandmarks.push_back(landmark.x);
> +			androidLandmarks.push_back(landmark.y);
> +		}
> +		resultMetadata->addEntry(
> +			ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
> +	}
> +
> +	const auto &faceDetectFaceIds =
> +		metadata.get(controls::draft::FaceDetectFaceIds);
> +	if (faceDetectRectangles && faceDetectFaceIds) {
> +		if (faceDetectFaceIds->size() != faceDetectRectangles->size()) {
> +			LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
> +					<< "Expected: "
> +					<< faceDetectRectangles->size()
> +					<< ", got: " << faceDetectFaceIds->size();
> +		}
> +		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds);
> +	}
> +
>  	const auto &scalerCrop = metadata.get(controls::ScalerCrop);
>  	if (scalerCrop) {
>  		const Rectangle &crop = *scalerCrop;
> --
> 2.46.0.792.g87dc391469-goog
>

Patch
diff mbox series 

diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
index 71043e12..a89a3115 100644
--- a/src/android/camera_capabilities.cpp
+++ b/src/android/camera_capabilities.cpp
@@ -7,6 +7,8 @@ 
 
 #include "camera_capabilities.h"
 
+#include <stdint.h>
+
 #include <algorithm>
 #include <array>
 #include <cmath>
@@ -1176,11 +1178,46 @@  int CameraCapabilities::initializeStaticMetadata()
 				  maxFrameDuration_);
 
 	/* Statistics static metadata. */
-	uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
-				  faceDetectMode);
-
 	int32_t maxFaceCount = 0;
+	auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
+	if (iter != camera_->controls().end()) {
+		const ControlInfo &faceDetectCtrlInfo = iter->second;
+		std::vector<uint8_t> faceDetectModes;
+		bool hasFaceDetection = false;
+
+		for (const auto &value : faceDetectCtrlInfo.values()) {
+			uint8_t mode = value.get<uint8_t>();
+			uint8_t androidMode = 0;
+
+			switch (mode) {
+			case controls::draft::FaceDetectModeOff:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+				break;
+			case controls::draft::FaceDetectModeSimple:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
+				hasFaceDetection = true;
+				break;
+			default:
+				LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
+			}
+			faceDetectModes.push_back(androidMode);
+		}
+		if (hasFaceDetection) {
+			/*
+			 * \todo Create new libcamera controls to query max
+			 * possible faces detected.
+			 */
+			maxFaceCount = 10;
+			staticMetadata_->addEntry(
+				ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+				faceDetectModes.data(), faceDetectModes.size());
+		}
+	} else {
+		uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+		staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+					  faceDetectMode);
+	}
+
 	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
 				  maxFaceCount);
 
diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index 493f66e7..e80b9e17 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -15,6 +15,7 @@ 
 #include <vector>
 
 #include <libcamera/base/log.h>
+#include <libcamera/base/span.h>
 #include <libcamera/base/unique_fd.h>
 #include <libcamera/base/utils.h>
 
@@ -22,6 +23,7 @@ 
 #include <libcamera/controls.h>
 #include <libcamera/fence.h>
 #include <libcamera/formats.h>
+#include <libcamera/geometry.h>
 #include <libcamera/property_ids.h>
 
 #include "system/graphics.h"
@@ -813,6 +815,11 @@  int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
 		controls.set(controls::ScalerCrop, cropRegion);
 	}
 
+	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
+		const uint8_t *data = entry.data.u8;
+		controls.set(controls::draft::FaceDetectMode, data[0]);
+	}
+
 	if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
 		const int32_t data = *entry.data.i32;
 		int32_t testPatternMode = controls::draft::TestPatternModeOff;
@@ -1540,8 +1547,9 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 	value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
 
-	value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
+	settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry);
+	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
+				 entry.data.u8, 1);
 
 	value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
@@ -1580,6 +1588,64 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 		resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
 					 *frameDuration * 1000);
 
+	const auto &faceDetectRectangles =
+		metadata.get(controls::draft::FaceDetectFaceRectangles);
+	if (faceDetectRectangles) {
+		std::vector<int32_t> flatRectangles;
+		for (const Rectangle &rect : *faceDetectRectangles) {
+			flatRectangles.push_back(rect.x);
+			flatRectangles.push_back(rect.y);
+			flatRectangles.push_back(rect.x + rect.width);
+			flatRectangles.push_back(rect.y + rect.height);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
+	}
+
+	const auto &faceDetectFaceScores =
+		metadata.get(controls::draft::FaceDetectFaceScores);
+	if (faceDetectRectangles && faceDetectFaceScores) {
+		if (faceDetectFaceScores->size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
+					<< "Expected: "
+					<< faceDetectRectangles->size()
+					<< ", got: " << faceDetectFaceScores->size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES,
+					 *faceDetectRectangles);
+	}
+
+	const auto &faceDetectFaceLandmarks =
+		metadata.get(controls::draft::FaceDetectFaceLandmarks);
+	if (faceDetectRectangles && faceDetectFaceLandmarks) {
+		size_t expectedLandmarks = faceDetectRectangles->size() * 3;
+		if (faceDetectFaceLandmarks->size() != expectedLandmarks) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
+					<< "Expected: " << expectedLandmarks
+					<< ", got: " << faceDetectFaceLandmarks->size();
+		}
+
+		std::vector<int32_t> androidLandmarks;
+		for (const auto &landmark : *faceDetectFaceLandmarks) {
+			androidLandmarks.push_back(landmark.x);
+			androidLandmarks.push_back(landmark.y);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
+	}
+
+	const auto &faceDetectFaceIds =
+		metadata.get(controls::draft::FaceDetectFaceIds);
+	if (faceDetectRectangles && faceDetectFaceIds) {
+		if (faceDetectFaceIds->size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
+					<< "Expected: "
+					<< faceDetectRectangles->size()
+					<< ", got: " << faceDetectFaceIds->size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds);
+	}
+
 	const auto &scalerCrop = metadata.get(controls::ScalerCrop);
 	if (scalerCrop) {
 		const Rectangle &crop = *scalerCrop;