From patchwork Mon Sep 23 09:30:46 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng-Hao Yang X-Patchwork-Id: 21333 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 4E145C0F1B for ; Mon, 23 Sep 2024 09:33:36 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id F037663511; Mon, 23 Sep 2024 11:33:35 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="lfINlWdX"; dkim-atps=neutral Received: from mail-pl1-x629.google.com (mail-pl1-x629.google.com [IPv6:2607:f8b0:4864:20::629]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7C67C63512 for ; Mon, 23 Sep 2024 11:33:29 +0200 (CEST) Received: by mail-pl1-x629.google.com with SMTP id d9443c01a7336-20543fdb7acso32164775ad.1 for ; Mon, 23 Sep 2024 02:33:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1727084007; x=1727688807; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=E/ouNh/kwG+GNdwoclRM6KB8psYsFquTmInQtIIqDoo=; b=lfINlWdX0uepLGFx2Rm0v+xBB7sADGcc1W93gpdKcv2vC6xaFDQSJXxrRvBTaeaqze oviDYh2zeCNpCD1VET3IdMJ3fLGkOvSYjmr9JbR3YR1qMH9HVF6+8sQ70rjJAi3CDxRn 7nP6vsfT+hWYdD/nYzxL+fHeov3Dxi99KW9GU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727084007; x=1727688807; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=E/ouNh/kwG+GNdwoclRM6KB8psYsFquTmInQtIIqDoo=; b=gg0Z8gQIf/NQOxS+/fBu8pu6nNupxIfbRF00L+UvWeP+Yz9dGZIdlZUjbGhaonfDKq Nv3gAkpInuczOeP46kmAraRuFL83fAa9H142ivecW6MU2oAYiYlwA/CDhJRGpXX31FTE 1jhTgH+fFGXI5rLWjq6+M1NtLWYSfIBKkedvsuINYCyGEANq2WdWSw2FKqlZ0H6p0i23 PJN/FQDDb+D0Id4XUfjtwI62M92rmQ+m6SLavId2Zzpkd8kDJzqPUdre0YSfZl6iuyz2 tG7E6VpHa1Ok2nGJFF++NGmiJ1mHKza7vhMO4iM9kzVQk3+muSfuSjdnnryCenVJt6xZ sQYA== X-Gm-Message-State: AOJu0YxXBzql14ST7zJ3R5UJfvAOP+xyWaSiN2xOHzQpNV+740hGiwpP FpFE0kKnEK/vI0MPBgy0kfCRlRLOjEac3mogxOuYPk4JxsKqWbJ3yiB6DRWaIDWh8U1biqiiQ4o = X-Google-Smtp-Source: AGHT+IEuwNS3OjMSCvrsgnygm4MnF/XedDDmd38zkxL/UQ4xfdkldOECJwo6L+sBzhv/w9l3kQE5sA== X-Received: by 2002:a17:902:cf4b:b0:200:869c:95e3 with SMTP id d9443c01a7336-208d8370e93mr191608335ad.4.1727084007163; Mon, 23 Sep 2024 02:33:27 -0700 (PDT) Received: from chenghaoyang-low.c.googlers.com.com (0.223.81.34.bc.googleusercontent.com. [34.81.223.0]) by smtp.gmail.com with ESMTPSA id d9443c01a7336-207946d6673sm129864995ad.158.2024.09.23.02.33.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 23 Sep 2024 02:33:26 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Harvey Yang , Yudhistira Erlandinata Subject: [PATCH v5 3/3] libcamera: android: Add face detection control support Date: Mon, 23 Sep 2024 09:30:46 +0000 Message-ID: <20240923093317.3500444-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.792.g87dc391469-goog In-Reply-To: <20240923093317.3500444-1-chenghaoyang@google.com> References: <20240923093317.3500444-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Harvey Yang Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang Reviewed-by: Jacopo Mondi --- src/android/camera_capabilities.cpp | 45 +++++++++++++++-- src/android/camera_device.cpp | 75 ++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e12..a89a3115 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -7,6 +7,8 @@ #include "camera_capabilities.h" +#include + #include #include #include @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + uint8_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::draft::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::draft::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7..50caa14a 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::draft::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1588,69 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::draft::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::draft::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::draft::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceScores && + faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = + metadata.get(controls::draft::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceScores && + faceDetectFaceLandmarks && faceDetectFaceIds) { + const Span &ids = *faceDetectFaceIds; + if (ids.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << ids.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;