From patchwork Fri Aug 23 13:07:35 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng-Hao Yang X-Patchwork-Id: 21001 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 99B8BC32C2 for ; Fri, 23 Aug 2024 13:12:01 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id CDA3A633D2; Fri, 23 Aug 2024 15:11:58 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="KNROJX9V"; dkim-atps=neutral Received: from mail-wm1-x334.google.com (mail-wm1-x334.google.com [IPv6:2a00:1450:4864:20::334]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 06A50633D2 for ; Fri, 23 Aug 2024 15:11:52 +0200 (CEST) Received: by mail-wm1-x334.google.com with SMTP id 5b1f17b1804b1-428fb103724so15987655e9.1 for ; Fri, 23 Aug 2024 06:11:52 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; t=1724418712; x=1725023512; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=TnlCiqzXEKYTAvjt8obOzNnSj3e1aMSJAms5MbC4p9E=; b=KNROJX9VDYn1MqmdWb6tJfEKX3ATDXXPSbealGk5wd6mIMT8ytgpmkHqklZGpCjh5k sSo9ZnpRXK3C56QzMfZZLNGcr1OXwP/eeIQT83iE01fHoKe/GaNb66Jxemtl8lWDVgZA CLKxSv07C/B3o4SdbeDLlfRfigbBlWKk8PEfc= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1724418712; x=1725023512; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=TnlCiqzXEKYTAvjt8obOzNnSj3e1aMSJAms5MbC4p9E=; b=VJPBzuko9VxOvdrhFdDMU2aOGwoQ1KYYmHPLJ+HQigEzOuuNm4/E5y1VV6RxmzL7xY jI+q35fw1RdHUDt+9pZ0opNEU/4HnqKolRUJkcodaJNY32uN7Q2ipOxMgOlbF1vsQfbF Hp9QZ3ri7MZjQwo13t1S+d6DayQbN0B/zIgRFObPvtt3E+SPdV1bbxGC3rz9Us1zGC5N 8lDda/juEc+lQauB2DlVQ2e6wjNNCaHyCj+FgovAw4crWU50TxCJdf76nALdqBkFY/2R cEjCNBQ1meGM/jFiBIwv0+Uw46FXuHCEseGWAXifOvGtglAJ9XmgDWOaYxG8xUtPZlFM 9vpg== X-Gm-Message-State: AOJu0YxVHo6tUzJ8HsFx7lbp864DWC52Wfip6t32KgSftP7uE1MwRqUB Gd45eAw86p5Hf+4CMXY/1m3rucQDiQsgaRwzRqxcLwqZo3svioLKhongXo9/WORndOHHqnlHkgA Tdods X-Google-Smtp-Source: AGHT+IHKsjZSymM2ZthRY07TdXxQk2zPHParmMfbiTyxmvCi8Y6JHkUOsqsIEjVIP5OxVbnGEW431Q== X-Received: by 2002:a5d:6e8b:0:b0:36b:5d86:d885 with SMTP id ffacd0b85a97d-373052b51a8mr3713078f8f.24.1724418711669; Fri, 23 Aug 2024 06:11:51 -0700 (PDT) Received: from chenghaoyang-germany.c.googlers.com.com (161.126.77.34.bc.googleusercontent.com. [34.77.126.161]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3730810ffb9sm4185000f8f.7.2024.08.23.06.11.51 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 23 Aug 2024 06:11:51 -0700 (PDT) From: Harvey Yang X-Google-Original-From: Harvey Yang To: libcamera-devel@lists.libcamera.org Cc: Yudhistira Erlandinata , Harvey Yang Subject: [PATCH v1 3/3] libcamera: android: Add face detection control support Date: Fri, 23 Aug 2024 13:07:35 +0000 Message-ID: <20240823131148.2449434-4-chenghaoyang@google.com> X-Mailer: git-send-email 2.46.0.295.g3b9ea8a38a-goog In-Reply-To: <20240823131148.2449434-1-chenghaoyang@google.com> References: <20240823131148.2449434-1-chenghaoyang@google.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Yudhistira Erlandinata Allow Android HAL adapter to pass the face detection metadata control to the pipeline and also send face detection metadata to the camera client if the pipeline generates it. BUG=b:308713855 TEST=emerge-geralt libcamera-mtkisp7 Signed-off-by: Yudhistira Erlandinata Co-developed-by: Harvey Yang --- src/android/camera_capabilities.cpp | 41 ++++++++++++++++-- src/android/camera_device.cpp | 64 +++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp index 71043e127..556edb471 100644 --- a/src/android/camera_capabilities.cpp +++ b/src/android/camera_capabilities.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -1176,11 +1177,43 @@ int CameraCapabilities::initializeStaticMetadata() maxFrameDuration_); /* Statistics static metadata. */ - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, - faceDetectMode); - int32_t maxFaceCount = 0; + if (camera_->controls().count(controls::FaceDetectMode.id()) > 0) { + const ControlInfo &faceDetectCtrlInfo = + camera_->controls().at(controls::FaceDetectMode.id()); + std::vector faceDetectModes; + bool hasFaceDetection = false; + for (const auto &value : faceDetectCtrlInfo.values()) { + auto mode = value.get(); + uint8_t androidMode = 0; + switch (mode) { + case controls::FaceDetectModeOff: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + break; + case controls::FaceDetectModeSimple: + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE; + hasFaceDetection = true; + break; + default: + LOG(HAL, Fatal) << "Received invalid face detect mode: " + << static_cast(mode); + } + faceDetectModes.push_back(androidMode); + } + if (hasFaceDetection) { + // todo(yerlandinata): Create new libcamera controls + // to query max possible faces detected. + maxFaceCount = 10; + staticMetadata_->addEntry( + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectModes.data(), faceDetectModes.size()); + } + } else { + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, + faceDetectMode); + } + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, maxFaceCount); diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 493f66e7b..cd600eade 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -9,12 +9,12 @@ #include #include -#include #include #include #include #include +#include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "system/graphics.h" @@ -813,6 +814,14 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor) controls.set(controls::ScalerCrop, cropRegion); } + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) { + const uint8_t *data = entry.data.u8; + controls.set(controls::FaceDetectMode, data[0]); + if (!controls.get(controls::FaceDetectMode)) { + LOG(HAL, Warning) << "Pipeline doesn't support controls::FaceDetectMode"; + } + } + if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) { const int32_t data = *entry.data.i32; int32_t testPatternMode = controls::draft::TestPatternModeOff; @@ -1540,8 +1549,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF; resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32); - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value); + settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry); + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, + entry.data.u8, 1); value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, @@ -1580,6 +1590,54 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION, *frameDuration * 1000); + const auto &faceDetectRectangles = + metadata.get(controls::FaceDetectFaceRectangles); + if (faceDetectRectangles) { + const Span rectangles = *faceDetectRectangles; + std::vector flatRectangles; + for (const Rectangle &rect : rectangles) { + flatRectangles.push_back(rect.x); + flatRectangles.push_back(rect.y); + flatRectangles.push_back(rect.x + rect.width); + flatRectangles.push_back(rect.y + rect.height); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles); + } + + const auto &faceDetectFaceScores = + metadata.get(controls::FaceDetectFaceScores); + if (faceDetectRectangles && faceDetectFaceScores) { + const Span &scores = *faceDetectFaceScores; + if (scores.size() != faceDetectRectangles->size()) { + LOG(HAL, Error) << "Pipeline returned wrong number of face scores; " + << "Expected: " + << faceDetectRectangles->size() + << ", got:" << scores.size(); + } + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores); + } + + const auto &faceDetectFaceLandmarks = + metadata.get(controls::FaceDetectFaceLandmark); + if (faceDetectFaceScores && faceDetectRectangles && + faceDetectFaceLandmarks) { + const auto &landmarks = *faceDetectFaceLandmarks; + size_t expectedLandmarks = faceDetectRectangles->size() * 3; + std::vector androidLandmarks; + for (const auto &landmark : landmarks) { + androidLandmarks.push_back(landmark.x); + androidLandmarks.push_back(landmark.y); + } + resultMetadata->addEntry( + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks); + if (landmarks.size() != expectedLandmarks) { + LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; " + << "Expected: " << expectedLandmarks + << ", got: " << landmarks.size(); + } + } + const auto &scalerCrop = metadata.get(controls::ScalerCrop); if (scalerCrop) { const Rectangle &crop = *scalerCrop;