[v9,5/5] libcamera: android: Add face detection control support
diff mbox series

Message ID 20240930195915.152187-6-jacopo.mondi@ideasonboard.com
State Accepted
Headers show
Series
  • Add Face Detection Controls
Related show

Commit Message

Jacopo Mondi Sept. 30, 2024, 7:59 p.m. UTC 
From: Harvey Yang <chenghaoyang@chromium.org>

Allow Android HAL adapter to pass the face detection metadata control to
the pipeline and also send face detection metadata to the camera client
if the pipeline generates it.

Signed-off-by: Yudhistira Erlandinata <yerlandinata@chromium.org>
Co-developed-by: Harvey Yang <chenghaoyang@chromium.org>
Signed-off-by: Harvey Yang <chenghaoyang@chromium.org>
Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
---
 src/android/camera_capabilities.cpp | 44 +++++++++++++++++--
 src/android/camera_device.cpp       | 66 ++++++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 6 deletions(-)

Comments

Harvey Yang Oct. 1, 2024, 7:17 a.m. UTC | #1 
Hi Jacopo,

On Tue, Oct 1, 2024 at 3:59 AM Jacopo Mondi
<jacopo.mondi@ideasonboard.com> wrote:
>
> From: Harvey Yang <chenghaoyang@chromium.org>
>
> Allow Android HAL adapter to pass the face detection metadata control to
> the pipeline and also send face detection metadata to the camera client
> if the pipeline generates it.
>
> Signed-off-by: Yudhistira Erlandinata <yerlandinata@chromium.org>
> Co-developed-by: Harvey Yang <chenghaoyang@chromium.org>
> Signed-off-by: Harvey Yang <chenghaoyang@chromium.org>
> Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
> Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
> ---
>  src/android/camera_capabilities.cpp | 44 +++++++++++++++++--
>  src/android/camera_device.cpp       | 66 ++++++++++++++++++++++++++++-
>  2 files changed, 104 insertions(+), 6 deletions(-)
>
> diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
> index 71043e127497..b161bc6b3ed6 100644
> --- a/src/android/camera_capabilities.cpp
> +++ b/src/android/camera_capabilities.cpp
> @@ -11,6 +11,7 @@
>  #include <array>
>  #include <cmath>
>  #include <map>
> +#include <stdint.h>
>  #include <type_traits>
>
>  #include <hardware/camera3.h>
> @@ -1176,11 +1177,46 @@ int CameraCapabilities::initializeStaticMetadata()
>                                   maxFrameDuration_);
>
>         /* Statistics static metadata. */
> -       uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> -       staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> -                                 faceDetectMode);
> -
>         int32_t maxFaceCount = 0;
> +       auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
> +       if (iter != camera_->controls().end()) {
> +               const ControlInfo &faceDetectCtrlInfo = iter->second;
> +               std::vector<uint8_t> faceDetectModes;
> +               bool hasFaceDetection = false;
> +
> +               for (const auto &value : faceDetectCtrlInfo.values()) {
> +                       int32_t mode = value.get<int32_t>();
> +                       uint8_t androidMode = 0;
> +
> +                       switch (mode) {
> +                       case controls::draft::FaceDetectModeOff:
> +                               androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> +                               break;
> +                       case controls::draft::FaceDetectModeSimple:
> +                               androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
> +                               hasFaceDetection = true;
> +                               break;
> +                       default:
> +                               LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
> +                       }
> +                       faceDetectModes.push_back(androidMode);
> +               }
> +               if (hasFaceDetection) {
> +                       /*
> +                        * \todo Create new libcamera controls to query max
> +                        * possible faces detected.
> +                        */
> +                       maxFaceCount = 10;
> +                       staticMetadata_->addEntry(
> +                               ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> +                               faceDetectModes.data(), faceDetectModes.size());
> +               }
> +       } else {
> +               uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> +               staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
> +                                         faceDetectMode);
> +       }
> +
>         staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
>                                   maxFaceCount);
>
> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
> index 493f66e7b38f..a038131ae39e 100644
> --- a/src/android/camera_device.cpp
> +++ b/src/android/camera_device.cpp
> @@ -22,6 +22,7 @@
>  #include <libcamera/controls.h>
>  #include <libcamera/fence.h>
>  #include <libcamera/formats.h>
> +#include <libcamera/geometry.h>
>  #include <libcamera/property_ids.h>
>
>  #include "system/graphics.h"
> @@ -813,6 +814,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
>                 controls.set(controls::ScalerCrop, cropRegion);
>         }
>
> +       if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
> +               const uint8_t *data = entry.data.u8;
> +               controls.set(controls::draft::FaceDetectMode, data[0]);
> +       }
> +
>         if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
>                 const int32_t data = *entry.data.i32;
>                 int32_t testPatternMode = controls::draft::TestPatternModeOff;
> @@ -1540,8 +1546,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
>         value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
>         resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
>
> -       value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
> -       resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
> +       if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry))
> +               resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
> +                                        entry.data.u8, 1);
>
>         value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
>         resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
> @@ -1580,6 +1587,61 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
>                 resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
>                                          *frameDuration * 1000);
>
> +       const auto &faceDetectRectangles =
> +               metadata.get(controls::draft::FaceDetectFaceRectangles);
> +       if (faceDetectRectangles) {
> +               std::vector<int32_t> flatRectangles;
> +               for (const Rectangle &rect : *faceDetectRectangles) {
> +                       flatRectangles.push_back(rect.x);
> +                       flatRectangles.push_back(rect.y);
> +                       flatRectangles.push_back(rect.x + rect.width);
> +                       flatRectangles.push_back(rect.y + rect.height);
> +               }
> +               resultMetadata->addEntry(
> +                       ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
> +       }
> +
> +       const auto &faceDetectFaceScores =
> +               metadata.get(controls::draft::FaceDetectFaceScores);
> +       if (faceDetectRectangles && faceDetectFaceScores) {
> +               if (faceDetectFaceScores->size() != faceDetectRectangles->size()) {
> +                       LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
> +                                       << "Expected: " << faceDetectRectangles->size()
> +                                       << ", got: " << faceDetectFaceScores->size();
> +               }
> +               resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES,
> +                                        *faceDetectFaceScores);

Thanks for fixing this!

Reviewed-by: Harvey Yang <chenghaoyang@chromium.org>

BR,
Harvey




> +       }
> +
> +       const auto &faceDetectFaceLandmarks =
> +               metadata.get(controls::draft::FaceDetectFaceLandmarks);
> +       if (faceDetectRectangles && faceDetectFaceLandmarks) {
> +               size_t expectedLandmarks = faceDetectRectangles->size() * 3;
> +               if (faceDetectFaceLandmarks->size() != expectedLandmarks) {
> +                       LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
> +                                       << "Expected: " << expectedLandmarks
> +                                       << ", got: " << faceDetectFaceLandmarks->size();
> +               }
> +
> +               std::vector<int32_t> androidLandmarks;
> +               for (const Point &landmark : *faceDetectFaceLandmarks) {
> +                       androidLandmarks.push_back(landmark.x);
> +                       androidLandmarks.push_back(landmark.y);
> +               }
> +               resultMetadata->addEntry(
> +                       ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
> +       }
> +
> +       const auto &faceDetectFaceIds = metadata.get(controls::draft::FaceDetectFaceIds);
> +       if (faceDetectRectangles && faceDetectFaceIds) {
> +               if (faceDetectFaceIds->size() != faceDetectRectangles->size()) {
> +                       LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
> +                                       << "Expected: " << faceDetectRectangles->size()
> +                                       << ", got: " << faceDetectFaceIds->size();
> +               }
> +               resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds);
> +       }
> +
>         const auto &scalerCrop = metadata.get(controls::ScalerCrop);
>         if (scalerCrop) {
>                 const Rectangle &crop = *scalerCrop;
> --
> 2.46.1
>

Patch
diff mbox series 

diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
index 71043e127497..b161bc6b3ed6 100644
--- a/src/android/camera_capabilities.cpp
+++ b/src/android/camera_capabilities.cpp
@@ -11,6 +11,7 @@ 
 #include <array>
 #include <cmath>
 #include <map>
+#include <stdint.h>
 #include <type_traits>
 
 #include <hardware/camera3.h>
@@ -1176,11 +1177,46 @@  int CameraCapabilities::initializeStaticMetadata()
 				  maxFrameDuration_);
 
 	/* Statistics static metadata. */
-	uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
-				  faceDetectMode);
-
 	int32_t maxFaceCount = 0;
+	auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
+	if (iter != camera_->controls().end()) {
+		const ControlInfo &faceDetectCtrlInfo = iter->second;
+		std::vector<uint8_t> faceDetectModes;
+		bool hasFaceDetection = false;
+
+		for (const auto &value : faceDetectCtrlInfo.values()) {
+			int32_t mode = value.get<int32_t>();
+			uint8_t androidMode = 0;
+
+			switch (mode) {
+			case controls::draft::FaceDetectModeOff:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+				break;
+			case controls::draft::FaceDetectModeSimple:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
+				hasFaceDetection = true;
+				break;
+			default:
+				LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
+			}
+			faceDetectModes.push_back(androidMode);
+		}
+		if (hasFaceDetection) {
+			/*
+			 * \todo Create new libcamera controls to query max
+			 * possible faces detected.
+			 */
+			maxFaceCount = 10;
+			staticMetadata_->addEntry(
+				ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+				faceDetectModes.data(), faceDetectModes.size());
+		}
+	} else {
+		uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+		staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+					  faceDetectMode);
+	}
+
 	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
 				  maxFaceCount);
 
diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index 493f66e7b38f..a038131ae39e 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -22,6 +22,7 @@ 
 #include <libcamera/controls.h>
 #include <libcamera/fence.h>
 #include <libcamera/formats.h>
+#include <libcamera/geometry.h>
 #include <libcamera/property_ids.h>
 
 #include "system/graphics.h"
@@ -813,6 +814,11 @@  int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
 		controls.set(controls::ScalerCrop, cropRegion);
 	}
 
+	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
+		const uint8_t *data = entry.data.u8;
+		controls.set(controls::draft::FaceDetectMode, data[0]);
+	}
+
 	if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
 		const int32_t data = *entry.data.i32;
 		int32_t testPatternMode = controls::draft::TestPatternModeOff;
@@ -1540,8 +1546,9 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 	value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
 
-	value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
+	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry))
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
+					 entry.data.u8, 1);
 
 	value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
@@ -1580,6 +1587,61 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 		resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
 					 *frameDuration * 1000);
 
+	const auto &faceDetectRectangles =
+		metadata.get(controls::draft::FaceDetectFaceRectangles);
+	if (faceDetectRectangles) {
+		std::vector<int32_t> flatRectangles;
+		for (const Rectangle &rect : *faceDetectRectangles) {
+			flatRectangles.push_back(rect.x);
+			flatRectangles.push_back(rect.y);
+			flatRectangles.push_back(rect.x + rect.width);
+			flatRectangles.push_back(rect.y + rect.height);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
+	}
+
+	const auto &faceDetectFaceScores =
+		metadata.get(controls::draft::FaceDetectFaceScores);
+	if (faceDetectRectangles && faceDetectFaceScores) {
+		if (faceDetectFaceScores->size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
+					<< "Expected: " << faceDetectRectangles->size()
+					<< ", got: " << faceDetectFaceScores->size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES,
+					 *faceDetectFaceScores);
+	}
+
+	const auto &faceDetectFaceLandmarks =
+		metadata.get(controls::draft::FaceDetectFaceLandmarks);
+	if (faceDetectRectangles && faceDetectFaceLandmarks) {
+		size_t expectedLandmarks = faceDetectRectangles->size() * 3;
+		if (faceDetectFaceLandmarks->size() != expectedLandmarks) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
+					<< "Expected: " << expectedLandmarks
+					<< ", got: " << faceDetectFaceLandmarks->size();
+		}
+
+		std::vector<int32_t> androidLandmarks;
+		for (const Point &landmark : *faceDetectFaceLandmarks) {
+			androidLandmarks.push_back(landmark.x);
+			androidLandmarks.push_back(landmark.y);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
+	}
+
+	const auto &faceDetectFaceIds = metadata.get(controls::draft::FaceDetectFaceIds);
+	if (faceDetectRectangles && faceDetectFaceIds) {
+		if (faceDetectFaceIds->size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
+					<< "Expected: " << faceDetectRectangles->size()
+					<< ", got: " << faceDetectFaceIds->size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds);
+	}
+
 	const auto &scalerCrop = metadata.get(controls::ScalerCrop);
 	if (scalerCrop) {
 		const Rectangle &crop = *scalerCrop;