From patchwork Mon Sep 23 14:45:05 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Harvey Yang X-Patchwork-Id: 21337 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id DD0C2C0F1B for ; Mon, 23 Sep 2024 14:49:39 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4F88663516; Mon, 23 Sep 2024 16:49:39 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="LWU6hEUc"; dkim-atps=neutral Received: from mail-pl1-x62c.google.com (mail-pl1-x62c.google.com [IPv6:2607:f8b0:4864:20::62c]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4ED0863516 for ; Mon, 23 Sep 2024 16:49:34 +0200 (CEST) Received: by mail-pl1-x62c.google.com with SMTP id d9443c01a7336-2053f6b8201so38761585ad.2 for ; Mon, 23 Sep 2024 07:49:34 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1727102972; x=1727707772; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=BBYIDOTmFw8r0qJaHkYpTkkfYv9NhuFdtJGkrKUY5TI=; b=LWU6hEUcG8Ftg5NwZpHAsvTRCvjEddx02zbMDameV3ZkDajv3KLgL9an3tHc39nABP rDDAQ6749c+ycjn6JTdCm2d7Vf5pkW6xYh6N7FcEVeXr/sqztbjIPRsnKFhKMiwuJA02 u+RSgHRIvQ2ZKKCftUiBi2rnU6/XaLKMfSpU0= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727102972; x=1727707772; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=BBYIDOTmFw8r0qJaHkYpTkkfYv9NhuFdtJGkrKUY5TI=; b=C6FaErqXNRX9JU2jvFf8Ug2Oou7NQQe/Lt+HatUfpSopK9W4kqO6ilSNwq1pdgJYC2 5Lgufn/xWB8scpjRdlpLD/gFeCYfBsXWIBJ2OThkcfzcUqyWTGI7rUMk+gMDLq02+JrR NH8WE75a9rZCYueHB18DzswEyhkakfJzrAIRTs/8n4RxSwFrDi9HBMeH10Po9I/qEYpW kCScjjpUR6zbzz/cxlwSJ2w8AxFVfVjBWiaU4cV1UJynuC2eMWVjii3vd4HimTd5hvAp lKuR46c3mXFokbNV7nFc5OKRP9hYZL2AZ4ACemDhC9cn2HYIzYz3chI6tkxAMG1tZDbq l7mg== X-Gm-Message-State: AOJu0YyNs+AZuqLJBjo3D1+hx0bCn+X45YZHBxQNdUP02QSx2jwLfSDw aPCHmXY6KyZac5Mn2Zg4kL2SsmVL1ZV0sT8AyW/8BjiU3Zn4bpDtx945o6Ovf8xzOBhU4R6OKfs = X-Google-Smtp-Source: AGHT+IHIL4/5WJ/a9330cWJ0osAR4WxUIbG6zmiquAIyxfvpyWPJS6e06LIBiiXvezr8/IKG6mknUA== X-Received: by 2002:a17:902:e5cf:b0:1fa:9c04:946a with SMTP id d9443c01a7336-208d97e3f62mr153307215ad.1.1727102972438; Mon, 23 Sep 2024 07:49:32 -0700 (PDT) Received: from chenghaoyang-low.c.googlers.com.com (0.223.81.34.bc.googleusercontent.com. [34.81.223.0]) by smtp.gmail.com with ESMTPSA id d9443c01a7336-20794735719sm133857435ad.275.2024.09.23.07.49.31 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 23 Sep 2024 07:49:32 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Harvey Yang , Yudhistira Erlandinata Subject: [PATCH v6 3/3] libcamera: android: Add face detection control support Date: Mon, 23 Sep 2024 14:45:05 +0000 Message-ID: <20240923144921.1729183-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.792.g87dc391469-goog In-Reply-To: <20240923144921.1729183-1-chenghaoyang@google.com> References: <20240923144921.1729183-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Harvey Yang Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang --- src/android/camera_capabilities.cpp | 45 ++++++++++++++++-- src/android/camera_device.cpp | 73 ++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e12..d52d58ed 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -7,6 +7,8 @@ #include "camera_capabilities.h" +#include + #include #include #include @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::android::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + uint8_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::android::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::android::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7..1982509d 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::android::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1588,67 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::android::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::android::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::android::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = + metadata.get(controls::android::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceIds) { + const Span &ids = *faceDetectFaceIds; + if (ids.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << ids.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;