From patchwork Wed Sep 25 08:12:26 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Harvey Yang X-Patchwork-Id: 21352 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D320EC0F1B for ; Wed, 25 Sep 2024 08:16:12 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 78FD863513; Wed, 25 Sep 2024 10:16:12 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="akeOn2os"; dkim-atps=neutral Received: from mail-pl1-x635.google.com (mail-pl1-x635.google.com [IPv6:2607:f8b0:4864:20::635]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 8AEA363513 for ; Wed, 25 Sep 2024 10:16:07 +0200 (CEST) Received: by mail-pl1-x635.google.com with SMTP id d9443c01a7336-206bd1c6ccdso61060965ad.3 for ; Wed, 25 Sep 2024 01:16:07 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1727252165; x=1727856965; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=wbC26ApX1s3jY2T8aakSTnoyjjRaO6JZdJgUgnelnAc=; b=akeOn2osiyvxQbFK98SndtCnRZQSkRTcwzRylm2T1IwKALAZWN5LTnvD2FhvWqxRPF sUlcnuecY0JgtGCo7gX1JWqgwrieED76D+Ib3cvMfDxU0uhY/QWxm9cTh+di0GSO8QMw 3InJGKXCcD83cALJs4x86H+X2xE2mX09ySkRY= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727252165; x=1727856965; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=wbC26ApX1s3jY2T8aakSTnoyjjRaO6JZdJgUgnelnAc=; b=Ncv60Q5OmtgXd9+CeANH79Pg1zZgyAvdx11bsSnnrQsnLpqIIsyDOA46oghcPz5ySY coywRHOXJgwDNTcaJeWzrcaHKMz2b4ziG4GK+dJbt3EYQ6sQW7NIKXpi7S3xm2y7OG4x zd3wUPdcpGfUit4hURb/l9LuKH0H8mUlFWGqAVyuOBs/CT6zPFy03FwyYf67XkMrPusI EuwO94qhLVH66Sz5CC7q7AOayh8uXyVK9SzIw1iESvReMrFKb1u2kN/s6DUfdeH1TC/E NLDSdAaeBUhduOznz+qxAnlmGr0eAZEAHcRuf7TSN9WWLwekfoN6t84ge7Z8mq2FEnET egzw== X-Gm-Message-State: AOJu0YwVzc+UKntBGBxVeOL7taHouwhwFm3xlf8U3t+wbQVszWg2OoVI w8BwLR93MvQlI3sJyq2l+EbZ+L4G5Eu65VTLLiZ+u8mX1fJDuu/knJfqkpP0h0dnUbMehjmkIS4 = X-Google-Smtp-Source: AGHT+IEWC5hzX5lmMv3nRzFu5mEiiHItJ+kI728bQJX+yVdV3kGw0UWZa1cLUgWsZmIjZKxV+EXC8A== X-Received: by 2002:a17:902:e54c:b0:1fb:90e1:c8c5 with SMTP id d9443c01a7336-20afc48670bmr28580705ad.33.1727252165506; Wed, 25 Sep 2024 01:16:05 -0700 (PDT) Received: from chenghaoyang-low.c.googlers.com.com (0.223.81.34.bc.googleusercontent.com. [34.81.223.0]) by smtp.gmail.com with ESMTPSA id d9443c01a7336-20b002aaa8fsm6336465ad.240.2024.09.25.01.16.03 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 25 Sep 2024 01:16:05 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Harvey Yang , Yudhistira Erlandinata , Jacopo Mondi Subject: [PATCH v8 3/3] libcamera: android: Add face detection control support Date: Wed, 25 Sep 2024 08:12:26 +0000 Message-ID: <20240925081551.3265942-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.792.g87dc391469-goog In-Reply-To: <20240925081551.3265942-1-chenghaoyang@google.com> References: <20240925081551.3265942-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Harvey Yang Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang Reviewed-by: Jacopo Mondi --- src/android/camera_capabilities.cpp | 45 +++++++++++++++++-- src/android/camera_device.cpp | 70 ++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e12..a89a3115 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -7,6 +7,8 @@ #include "camera_capabilities.h" +#include + #include #include #include @@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + + for (const auto &value : faceDetectCtrlInfo.values()) { + uint8_t mode = value.get(); + uint8_t androidMode = 0; + + switch (mode) { + case controls::draft::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::draft::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode; + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + /* + * \todo Create new libcamera controls to query max + * possible faces detected. + */ + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7..e80b9e17 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::draft::FaceDetectMode, data[0]); + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1588,64 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::draft::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + std::vector flatRectangles; + for (const Rectangle &rect : *faceDetectRectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::draft::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + if (faceDetectFaceScores->size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << faceDetectFaceScores->size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, + *faceDetectRectangles); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::draft::FaceDetectFaceLandmarks); + if (faceDetectRectangles && faceDetectFaceLandmarks) { + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + if (faceDetectFaceLandmarks->size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << faceDetectFaceLandmarks->size(); + } + + std::vector androidLandmarks; + for (const auto &landmark : *faceDetectFaceLandmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + } + + const auto &faceDetectFaceIds = + metadata.get(controls::draft::FaceDetectFaceIds); + if (faceDetectRectangles && faceDetectFaceIds) { + if (faceDetectFaceIds->size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face ids; " + << "Expected: " + << faceDetectRectangles->size() + << ", got: " << faceDetectFaceIds->size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds); + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;