From patchwork Tue Sep 24 11:02:46 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng-Hao Yang X-Patchwork-Id: 21345 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 3E550C32D4 for ; Tue, 24 Sep 2024 11:04:11 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id A01AA63512; Tue, 24 Sep 2024 13:04:10 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="L8IUGEpz"; dkim-atps=neutral Received: from mail-pf1-x433.google.com (mail-pf1-x433.google.com [IPv6:2607:f8b0:4864:20::433]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 89D3A63515 for ; Tue, 24 Sep 2024 13:04:05 +0200 (CEST) Received: by mail-pf1-x433.google.com with SMTP id d2e1a72fcca58-718d6ad6050so3931280b3a.0 for ; Tue, 24 Sep 2024 04:04:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1727175844; x=1727780644; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=BBYIDOTmFw8r0qJaHkYpTkkfYv9NhuFdtJGkrKUY5TI=; b=L8IUGEpzFBpV2XzXDjDNsKa97jEB8ABfCaJ/+gZFv+zUcQSxDiz1Hin9m553mWdrtQ Yo+DjFze4LrpiqDSuOZSyG5aTcGPyZpiKIddXUokqhmqNwnHOxk5mAXJiUG5oPyx+Q/+ pVuQ9Dtvln2Y07Dcc4t4ahRF8N5aijzlXLUBM= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727175844; x=1727780644; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=BBYIDOTmFw8r0qJaHkYpTkkfYv9NhuFdtJGkrKUY5TI=; b=ixkUEbsdlEm6NIcnLvVdLmZ2gd9XjnhDRuwO91T+TkgxJy/Bk0XeINtE7O3CIBSoV4 V1RvU9X7dwT02w59rJc8YB1a1SitkN74YQmzyIRfWBO2oCVI2aciP9snJ5OKWJNxpxid pyL3Nrvn9ArUgsNzszsQZ0AymdRfn3J5cJrmlx7y6MPpgiRdzfFyZcmQb3KVG73lL6cL 6rko2yZ0mhzt9TtioLhV4IxRFMYd0q7UGZYiaOCyPR9/WYr/x/NMjBZGWj/9PjKTfQ+8 VQLjc8y8X3pvTzVSqfrTVBRv/Dr7ONeelAdOfF0kCQQBB3Owk591ike+xAkUSu7roXDB U3Ow== X-Gm-Message-State: AOJu0Yxu+KeXSlvq4Gc+FilqxophPbjz56+wTBVantkYjigm+hXA1Mps EsXzw1htTFpz7kLsi6kgjopOFV8bQiPj/IyBSlsUFkj9TJN7xQC8Pv6MQlWVUoRBm5lr+x3ClJ0 = X-Google-Smtp-Source: AGHT+IHGxkzGqX3jAyLqjY6/uBOHXHKy2YN4yFmL8sPlSehS69zefcGvduFZhabDsEhyTpk3ox72JQ== X-Received: by 2002:a05:6a00:b92:b0:70d:21d9:e2ae with SMTP id d2e1a72fcca58-7199c9393d1mr24946490b3a.6.1727175843451; Tue, 24 Sep 2024 04:04:03 -0700 (PDT) Received: from chenghaoyang-low.c.googlers.com.com (208.158.221.35.bc.googleusercontent.com. [35.221.158.208]) by smtp.gmail.com with ESMTPSA id d2e1a72fcca58-71afc9833c3sm993326b3a.182.2024.09.24.04.04.01 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 24 Sep 2024 04:04:03 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Harvey Yang , Yudhistira Erlandinata Subject: [PATCH v7 3/3] libcamera: android: Add face detection control support Date: Tue, 24 Sep 2024 11:02:46 +0000 Message-ID: <20240924110353.2025397-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.792.g87dc391469-goog In-Reply-To: <20240924110353.2025397-1-chenghaoyang@google.com> References: <20240924110353.2025397-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Harvey Yang Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang Reviewed-by: Jacopo Mondi --- src/android/camera_capabilities.cpp | 45 ++++++++++++++++-- src/android/camera_device.cpp | 73 ++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e12..d52d58ed 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -7,6 +7,8 @@ #include "camera_capabilities.h" +#include + #include #include #include @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::android::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + uint8_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::android::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::android::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7..1982509d 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::android::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1588,67 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::android::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::android::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::android::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = + metadata.get(controls::android::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceIds) { + const Span &ids = *faceDetectFaceIds; + if (ids.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << ids.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;