From patchwork Mon Sep 30 19:59:13 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 21443 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 8BF1DC3257 for ; Mon, 30 Sep 2024 19:59:46 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id A43866351C; Mon, 30 Sep 2024 21:59:45 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Z2DtUP4C"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id BCFE463519 for ; Mon, 30 Sep 2024 21:59:35 +0200 (CEST) Received: from ideasonboard.com (unknown [95.131.46.153]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 15E112302; Mon, 30 Sep 2024 21:58:03 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1727726284; bh=sPHQzatYw/HYcmBUg3qDI6+GLHmbXeFy/S5N85m61so=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Z2DtUP4CjI5z/Nkg7Kk6vdoCm9IuFLDtx22oAhUGxmsyrT8iPtFiSjjSnFdfSpH33 5bB8VyiNGnut+1yDbvUcuoirFLSBWj9I3iaPGKNyJS0dm/bR2z+IFI2pEanA9JYWe3 mhFf+qCc0qQUVh+mY5FJjscTmWcJQAJaHKv/ESnQ= From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Cc: Jacopo Mondi , Harvey Yang , Harvey Yang Subject: [PATCH v9 5/5] libcamera: android: Add face detection control support Date: Mon, 30 Sep 2024 21:59:13 +0200 Message-ID: <20240930195915.152187-6-jacopo.mondi@ideasonboard.com> X-Mailer: git-send-email 2.46.1 In-Reply-To: <20240930195915.152187-1-jacopo.mondi@ideasonboard.com> References: <20240930195915.152187-1-jacopo.mondi@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Harvey Yang Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang Signed-off-by: Harvey Yang Reviewed-by: Jacopo Mondi Signed-off-by: Jacopo Mondi Reviewed-by: Harvey Yang --- src/android/camera_capabilities.cpp | 44 +++++++++++++++++-- src/android/camera_device.cpp | 66 ++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e127497..b161bc6b3ed6 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -1176,11 +1177,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + int32_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::draft::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::draft::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7b38f..a038131ae39e 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +814,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::draft::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1546,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1587,61 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::draft::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + std::vector flatRectangles; + for (const Rectangle &rect : *faceDetectRectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::draft::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + if (faceDetectFaceScores->size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " << faceDetectRectangles->size() + << ", got: " << faceDetectFaceScores->size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, + *faceDetectFaceScores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::draft::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceLandmarks) { + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (faceDetectFaceLandmarks->size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << faceDetectFaceLandmarks->size(); + } + + std::vector androidLandmarks; + for (const Point &landmark : *faceDetectFaceLandmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = metadata.get(controls::draft::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceIds) { + if (faceDetectFaceIds->size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " << faceDetectRectangles->size() + << ", got: " << faceDetectFaceIds->size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;