[v4,3/3] libcamera: android: Add face detection control support
diff mbox series

Message ID 20240903104018.3289112-4-chenghaoyang@chromium.org
State Superseded
Headers show
Series
  • Add Face Detection Controls
Related show

Commit Message

Cheng-Hao Yang Sept. 3, 2024, 10:39 a.m. UTC 
Allow Android HAL adapter to pass the face detection metadata control to
the pipeline and also send face detection metadata to the camera client
if the pipeline generates it.

Signed-off-by: Yudhistira Erlandinata <yerlandinata@chromium.org>
Co-developed-by: Harvey Yang <chenghaoyang@chromium.org>
---
 src/android/camera_capabilities.cpp | 45 +++++++++++++++--
 src/android/camera_device.cpp       | 75 ++++++++++++++++++++++++++++-
 2 files changed, 114 insertions(+), 6 deletions(-)

Patch
diff mbox series 

diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
index 71043e127..a89a3115d 100644
--- a/src/android/camera_capabilities.cpp
+++ b/src/android/camera_capabilities.cpp
@@ -7,6 +7,8 @@ 
 
 #include "camera_capabilities.h"
 
+#include <stdint.h>
+
 #include <algorithm>
 #include <array>
 #include <cmath>
@@ -1176,11 +1178,46 @@  int CameraCapabilities::initializeStaticMetadata()
 				  maxFrameDuration_);
 
 	/* Statistics static metadata. */
-	uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
-				  faceDetectMode);
-
 	int32_t maxFaceCount = 0;
+	auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
+	if (iter != camera_->controls().end()) {
+		const ControlInfo &faceDetectCtrlInfo = iter->second;
+		std::vector<uint8_t> faceDetectModes;
+		bool hasFaceDetection = false;
+
+		for (const auto &value : faceDetectCtrlInfo.values()) {
+			uint8_t mode = value.get<uint8_t>();
+			uint8_t androidMode = 0;
+
+			switch (mode) {
+			case controls::draft::FaceDetectModeOff:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+				break;
+			case controls::draft::FaceDetectModeSimple:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
+				hasFaceDetection = true;
+				break;
+			default:
+				LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
+			}
+			faceDetectModes.push_back(androidMode);
+		}
+		if (hasFaceDetection) {
+			/*
+			 * \todo Create new libcamera controls to query max
+			 * possible faces detected.
+			 */
+			maxFaceCount = 10;
+			staticMetadata_->addEntry(
+				ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+				faceDetectModes.data(), faceDetectModes.size());
+		}
+	} else {
+		uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+		staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+					  faceDetectMode);
+	}
+
 	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
 				  maxFaceCount);
 
diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index 493f66e7b..50caa14a9 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -15,6 +15,7 @@ 
 #include <vector>
 
 #include <libcamera/base/log.h>
+#include <libcamera/base/span.h>
 #include <libcamera/base/unique_fd.h>
 #include <libcamera/base/utils.h>
 
@@ -22,6 +23,7 @@ 
 #include <libcamera/controls.h>
 #include <libcamera/fence.h>
 #include <libcamera/formats.h>
+#include <libcamera/geometry.h>
 #include <libcamera/property_ids.h>
 
 #include "system/graphics.h"
@@ -813,6 +815,11 @@  int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
 		controls.set(controls::ScalerCrop, cropRegion);
 	}
 
+	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
+		const uint8_t *data = entry.data.u8;
+		controls.set(controls::draft::FaceDetectMode, data[0]);
+	}
+
 	if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
 		const int32_t data = *entry.data.i32;
 		int32_t testPatternMode = controls::draft::TestPatternModeOff;
@@ -1540,8 +1547,9 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 	value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
 
-	value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
+	settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry);
+	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
+				 entry.data.u8, 1);
 
 	value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
@@ -1580,6 +1588,69 @@  CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 		resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
 					 *frameDuration * 1000);
 
+	const auto &faceDetectRectangles =
+		metadata.get(controls::draft::FaceDetectFaceRectangles);
+	if (faceDetectRectangles) {
+		const Span<const Rectangle> rectangles = *faceDetectRectangles;
+		std::vector<int32_t> flatRectangles;
+		for (const Rectangle &rect : rectangles) {
+			flatRectangles.push_back(rect.x);
+			flatRectangles.push_back(rect.y);
+			flatRectangles.push_back(rect.x + rect.width);
+			flatRectangles.push_back(rect.y + rect.height);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
+	}
+
+	const auto &faceDetectFaceScores =
+		metadata.get(controls::draft::FaceDetectFaceScores);
+	if (faceDetectRectangles && faceDetectFaceScores) {
+		const Span<const uint8_t> &scores = *faceDetectFaceScores;
+		if (scores.size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
+					<< "Expected: "
+					<< faceDetectRectangles->size()
+					<< ", got:" << scores.size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores);
+	}
+
+	const auto &faceDetectFaceLandmarks =
+		metadata.get(controls::draft::FaceDetectFaceLandmarks);
+	if (faceDetectRectangles && faceDetectFaceScores &&
+	    faceDetectFaceLandmarks) {
+		const auto &landmarks = *faceDetectFaceLandmarks;
+		size_t expectedLandmarks = faceDetectRectangles->size() * 3;
+		if (landmarks.size() != expectedLandmarks) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
+					<< "Expected: " << expectedLandmarks
+					<< ", got: " << landmarks.size();
+		}
+
+		std::vector<int32_t> androidLandmarks;
+		for (const auto &landmark : landmarks) {
+			androidLandmarks.push_back(landmark.x);
+			androidLandmarks.push_back(landmark.y);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
+	}
+
+	const auto &faceDetectFaceIds =
+		metadata.get(controls::draft::FaceDetectFaceIds);
+	if (faceDetectRectangles && faceDetectFaceScores &&
+	    faceDetectFaceLandmarks && faceDetectFaceIds) {
+		const Span<const int32_t> &ids = *faceDetectFaceIds;
+		if (ids.size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
+					<< "Expected: "
+					<< faceDetectRectangles->size()
+					<< ", got:" << ids.size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids);
+	}
+
 	const auto &scalerCrop = metadata.get(controls::ScalerCrop);
 	if (scalerCrop) {
 		const Rectangle &crop = *scalerCrop;