From patchwork Fri Aug 23 14:29:10 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng-Hao Yang X-Patchwork-Id: 21005 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 64BA8BDE17 for ; Fri, 23 Aug 2024 14:32:21 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 78CD5633D7; Fri, 23 Aug 2024 16:32:18 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="dBv1GOI7"; dkim-atps=neutral Received: from mail-wr1-x42e.google.com (mail-wr1-x42e.google.com [IPv6:2a00:1450:4864:20::42e]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id DB11E633CF for ; Fri, 23 Aug 2024 16:32:10 +0200 (CEST) Received: by mail-wr1-x42e.google.com with SMTP id ffacd0b85a97d-3719f0758c6so1109981f8f.1 for ; Fri, 23 Aug 2024 07:32:10 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1724423530; x=1725028330; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=aH7hiKimyPigrNoK5TEoeqT1W2nUv57fQS07j7w7vd0=; b=dBv1GOI7/gRggAFYI+0y5ieA4aTnfRwCdvib+XzSba0he2eMtzNDX0P/DxShqndDO4 NTf2o3pxHqZMD99v0wVGywhAVCV18MacxE08XYUibd+HCC4/CLzG+Ni2r7LTzTndGaRb 5tzFk1cs/F83myTlv5BNW+CfvA1bqS+I4TibU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1724423530; x=1725028330; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=aH7hiKimyPigrNoK5TEoeqT1W2nUv57fQS07j7w7vd0=; b=JJrM64XFzfE3HkAcMNbstI5EHOo/7kJn7w3SY73XuS96MmuLXffZl/8T1i6yaeRS5q VOnlU4SofcMSvXB0yUXRLmP560rwpiIpFFhfqNxQMaY6baXf6v9M1oC+RYe2MG2mrrpe yromd26ELDDClFuqo1YuM/gdEcvhIo8z9+3nLqyqyrBVXQUtw7KvD2YAmN+rzDbuZQ/S lNfMKilZBsV+qXY9Q+1TBfZSj0CBAUL36fBXv9DQi/bO21uN5Hb+sQ8u/XgdwXox42pN 9m8Yd1bkKBw/H4lihVcefDnvT3dipm2bXX3nKUXQ5jnMR22GAAkRhDC1PMtOg7y3B0o6 Rjgg== X-Gm-Message-State: AOJu0Yx/4ye4ps0On8MswUDbR95celMfT2xeyuicalD420bZKE5yD44V xt/hMDhoTH7vaMAlLa2LlT7yXmNuMZpSfOGJFfgXYh4wXgJvtO2mciHDrHrAwUfoRkaQYWD0Br0 6Rg== X-Google-Smtp-Source: AGHT+IFqeKDVYybOSYwXSsdkvthCF9O0KgiFc06MqAkSozNwUAv5doLD9IxhVyPxR7a1/Q6RY+h1vA== X-Received: by 2002:adf:b19c:0:b0:368:37e3:dff7 with SMTP id ffacd0b85a97d-37311863b94mr1701776f8f.34.1724423530095; Fri, 23 Aug 2024 07:32:10 -0700 (PDT) Received: from chenghaoyang-germany.c.googlers.com.com (161.126.77.34.bc.googleusercontent.com. [34.77.126.161]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-373081ffb66sm4300627f8f.84.2024.08.23.07.32.09 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 23 Aug 2024 07:32:09 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Yudhistira Erlandinata , Harvey Yang Subject: [PATCH v2 3/3] libcamera: android: Add face detection control support Date: Fri, 23 Aug 2024 14:29:10 +0000 Message-ID: <20240823143205.2668765-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.295.g3b9ea8a38a-goog In-Reply-To: <20240823143205.2668765-1-chenghaoyang@google.com> References: <20240823143205.2668765-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Yudhistira Erlandinata Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. Squashed CL: https://chromium-review.googlesource.com/c/chromiumos/third_party/libcamera/+/5065292 Don't use ControlInfoMap::count() and ControlInfoMap::at() because its internal assumption is wrong: The ControlIdMap and its idmap have a 1:1 mapping between their entries. BUG=b:308713855 TEST=emerge-geralt libcamera-mtkisp7 Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang --- src/android/camera_capabilities.cpp | 41 ++++++++++++++++-- src/android/camera_device.cpp | 64 +++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e127..31d113d51 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -1176,11 +1177,43 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + auto iter = camera_->controls().find(controls::FaceDetectMode.id()); + if (iter != camera_->controls().end()) { + const ControlInfo &faceDetectCtrlInfo = iter->second; + std::vector faceDetectModes; + bool hasFaceDetection = false; + for (const auto &value : faceDetectCtrlInfo.values()) { + auto mode = value.get(); + uint8_t androidMode = 0; + switch (mode) { + case controls::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " + << static_cast(mode); + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + // todo(yerlandinata): Create new libcamera controls + // to query max possible faces detected. + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7b..cd600eade 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -9,12 +9,12 @@ #include #include -#include #include #include #include #include +#include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +814,14 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::FaceDetectMode, data[0]); + if (!controls.get(controls::FaceDetectMode)) { + LOG(HAL, Warning) << "Pipeline doesn't support controls::FaceDetectMode"; + } + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1549,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1590,54 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::FaceDetectFaceLandmark); + if (faceDetectFaceScores && faceDetectRectangles && + faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;