From patchwork Tue Sep 3 10:39:19 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng-Hao Yang X-Patchwork-Id: 21156 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C7362BF415 for ; Tue, 3 Sep 2024 10:40:30 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3E1AB634F0; Tue, 3 Sep 2024 12:40:28 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="djv65QAW"; dkim-atps=neutral Received: from mail-wm1-x332.google.com (mail-wm1-x332.google.com [IPv6:2a00:1450:4864:20::332]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 004FC634E7 for ; Tue, 3 Sep 2024 12:40:23 +0200 (CEST) Received: by mail-wm1-x332.google.com with SMTP id 5b1f17b1804b1-42bbd16fcf2so34960915e9.2 for ; Tue, 03 Sep 2024 03:40:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1725360023; x=1725964823; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=w5jvf2XJHzC4o9iH56eCt9R8b2ZHy6x6b/geIgEwN2s=; b=djv65QAW61321Fk/pfVlJndeL5xvfC9FQoqNBDqqg/L4FCU8HhKwuSE2bK/UuIH8Lj LyXBTJABuXeFHT7DVPZIJNvXoyw213N2gkPJOwOqkD2gWQ3wnoOZaNhVIieYwiPoSGMO R+NA8B8KBINneFTrj5PXymNvYwsJHTHEVYCJk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1725360023; x=1725964823; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=w5jvf2XJHzC4o9iH56eCt9R8b2ZHy6x6b/geIgEwN2s=; b=BDNIvOLn+wvqCaYIr9ufVShrs8b7xsb8bfh1i6Am7bpJVwPSCZJLlIeyxqgXWQDXE7 Olh+P2y6Emj6i1Q6dKdk1OHpFmoHii9SsWGliniQw6cWOn+L8OyyIuPAPBA61PuSEauS CNEME8zjeFEqHuddPTWdOJSXyVsfcleU4cNUExue75Dyl3+QErp8hlElU3wi05VL9dKH 42k+SWSi0pw6nq/NPX+6rF+VIaWSlUnuU5PuyHLMI+GHpb3e8ruv3KTawblPiwncWC3f WHXCPrYfyMDSBHAbgZxL3KRSWVnWHc9Q1UHHhVrH6JgIbw4pbDh9cBYY+Di1A7w/g6/j rO0g== X-Gm-Message-State: AOJu0YxamHQEfyHHBxj1NupXuB63JFEYW9M4/jG7MAGmUgz78Uc2crtE GW94MeaOr1sCodHBhamw24curB5ub/VLPjJVGM1gomoaMxwjYK3I32KYYEGvGBEqY8wOe+XJPlE Niw== X-Google-Smtp-Source: AGHT+IEsm/yieETdIr1Kz3c+oaFXmTGt5MVPJYtypxmvbi90nJHkXtwAr8vHhlgtRTVUn8ytyVY/5Q== X-Received: by 2002:a05:600c:5110:b0:424:ad14:6b79 with SMTP id 5b1f17b1804b1-42bb01aa1f2mr122620705e9.8.1725360022990; Tue, 03 Sep 2024 03:40:22 -0700 (PDT) Received: from chenghaoyang-germany.c.googlers.com.com (161.126.77.34.bc.googleusercontent.com. [34.77.126.161]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-42bb6deb239sm170546025e9.5.2024.09.03.03.40.22 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 03 Sep 2024 03:40:22 -0700 (PDT) From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Harvey Yang , Yudhistira Erlandinata Subject: [PATCH v4 3/3] libcamera: android: Add face detection control support Date: Tue, 3 Sep 2024 10:39:19 +0000 Message-ID: <20240903104018.3289112-4-chenghaoyang@chromium.org> X-Mailer: git-send-email 2.46.0.469.g59c65b2a67-goog In-Reply-To: <20240903104018.3289112-1-chenghaoyang@chromium.org> References: <20240903104018.3289112-1-chenghaoyang@chromium.org> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang --- src/android/camera_capabilities.cpp | 45 +++++++++++++++-- src/android/camera_device.cpp | 75 ++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e127..a89a3115d 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -7,6 +7,8 @@ #include "camera_capabilities.h" +#include + #include #include #include @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + uint8_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::draft::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::draft::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7b..50caa14a9 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::draft::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1588,69 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::draft::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::draft::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::draft::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceScores && + faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = + metadata.get(controls::draft::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceScores && + faceDetectFaceLandmarks && faceDetectFaceIds) { + const Span &ids = *faceDetectFaceIds; + if (ids.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << ids.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;