[{"id":31482,"web_url":"https://patchwork.libcamera.org/comment/31482/","msgid":"","date":"2024-10-01T07:17:58","subject":"Re: [PATCH v9 5/5] libcamera: android: Add face detection control\n\tsupport","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Tue, Oct 1, 2024 at 3:59 AM Jacopo Mondi\n wrote:\n>\n> From: Harvey Yang \n>\n> Allow Android HAL adapter to pass the face detection metadata control to\n> the pipeline and also send face detection metadata to the camera client\n> if the pipeline generates it.\n>\n> Signed-off-by: Yudhistira Erlandinata \n> Co-developed-by: Harvey Yang \n> Signed-off-by: Harvey Yang \n> Reviewed-by: Jacopo Mondi \n> Signed-off-by: Jacopo Mondi \n> ---\n> src/android/camera_capabilities.cpp | 44 +++++++++++++++++--\n> src/android/camera_device.cpp | 66 ++++++++++++++++++++++++++++-\n> 2 files changed, 104 insertions(+), 6 deletions(-)\n>\n> diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp\n> index 71043e127497..b161bc6b3ed6 100644\n> --- a/src/android/camera_capabilities.cpp\n> +++ b/src/android/camera_capabilities.cpp\n> @@ -11,6 +11,7 @@\n> #include \n> #include \n> #include \n> +#include \n> #include \n>\n> #include \n> @@ -1176,11 +1177,46 @@ int CameraCapabilities::initializeStaticMetadata()\n> maxFrameDuration_);\n>\n> /* Statistics static metadata. */\n> - uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;\n> - staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,\n> - faceDetectMode);\n> -\n> int32_t maxFaceCount = 0;\n> + auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());\n> + if (iter != camera_->controls().end()) {\n> + const ControlInfo &faceDetectCtrlInfo = iter->second;\n> + std::vector faceDetectModes;\n> + bool hasFaceDetection = false;\n> +\n> + for (const auto &value : faceDetectCtrlInfo.values()) {\n> + int32_t mode = value.get();\n> + uint8_t androidMode = 0;\n> +\n> + switch (mode) {\n> + case controls::draft::FaceDetectModeOff:\n> + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;\n> + break;\n> + case controls::draft::FaceDetectModeSimple:\n> + androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;\n> + hasFaceDetection = true;\n> + break;\n> + default:\n> + LOG(HAL, Fatal) << \"Received invalid face detect mode: \" << mode;\n> + }\n> + faceDetectModes.push_back(androidMode);\n> + }\n> + if (hasFaceDetection) {\n> + /*\n> + * \\todo Create new libcamera controls to query max\n> + * possible faces detected.\n> + */\n> + maxFaceCount = 10;\n> + staticMetadata_->addEntry(\n> + ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,\n> + faceDetectModes.data(), faceDetectModes.size());\n> + }\n> + } else {\n> + uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;\n> + staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,\n> + faceDetectMode);\n> + }\n> +\n> staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,\n> maxFaceCount);\n>\n> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> index 493f66e7b38f..a038131ae39e 100644\n> --- a/src/android/camera_device.cpp\n> +++ b/src/android/camera_device.cpp\n> @@ -22,6 +22,7 @@\n> #include \n> #include \n> #include \n> +#include \n> #include \n>\n> #include \"system/graphics.h\"\n> @@ -813,6 +814,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)\n> controls.set(controls::ScalerCrop, cropRegion);\n> }\n>\n> + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {\n> + const uint8_t *data = entry.data.u8;\n> + controls.set(controls::draft::FaceDetectMode, data[0]);\n> + }\n> +\n> if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {\n> const int32_t data = *entry.data.i32;\n> int32_t testPatternMode = controls::draft::TestPatternModeOff;\n> @@ -1540,8 +1546,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons\n> value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;\n> resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);\n>\n> - value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;\n> - resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);\n> + if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry))\n> + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,\n> + entry.data.u8, 1);\n>\n> value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;\n> resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,\n> @@ -1580,6 +1587,61 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons\n> resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,\n> *frameDuration * 1000);\n>\n> + const auto &faceDetectRectangles =\n> + metadata.get(controls::draft::FaceDetectFaceRectangles);\n> + if (faceDetectRectangles) {\n> + std::vector flatRectangles;\n> + for (const Rectangle &rect : *faceDetectRectangles) {\n> + flatRectangles.push_back(rect.x);\n> + flatRectangles.push_back(rect.y);\n> + flatRectangles.push_back(rect.x + rect.width);\n> + flatRectangles.push_back(rect.y + rect.height);\n> + }\n> + resultMetadata->addEntry(\n> + ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);\n> + }\n> +\n> + const auto &faceDetectFaceScores =\n> + metadata.get(controls::draft::FaceDetectFaceScores);\n> + if (faceDetectRectangles && faceDetectFaceScores) {\n> + if (faceDetectFaceScores->size() != faceDetectRectangles->size()) {\n> + LOG(HAL, Error) << \"Pipeline returned wrong number of face scores; \"\n> + << \"Expected: \" << faceDetectRectangles->size()\n> + << \", got: \" << faceDetectFaceScores->size();\n> + }\n> + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES,\n> + *faceDetectFaceScores);\n\nThanks for fixing this!\n\nReviewed-by: Harvey Yang \n\nBR,\nHarvey\n\n\n\n\n> + }\n> +\n> + const auto &faceDetectFaceLandmarks =\n> + metadata.get(controls::draft::FaceDetectFaceLandmarks);\n> + if (faceDetectRectangles && faceDetectFaceLandmarks) {\n> + size_t expectedLandmarks = faceDetectRectangles->size() * 3;\n> + if (faceDetectFaceLandmarks->size() != expectedLandmarks) {\n> + LOG(HAL, Error) << \"Pipeline returned wrong number of face landmarks; \"\n> + << \"Expected: \" << expectedLandmarks\n> + << \", got: \" << faceDetectFaceLandmarks->size();\n> + }\n> +\n> + std::vector androidLandmarks;\n> + for (const Point &landmark : *faceDetectFaceLandmarks) {\n> + androidLandmarks.push_back(landmark.x);\n> + androidLandmarks.push_back(landmark.y);\n> + }\n> + resultMetadata->addEntry(\n> + ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);\n> + }\n> +\n> + const auto &faceDetectFaceIds = metadata.get(controls::draft::FaceDetectFaceIds);\n> + if (faceDetectRectangles && faceDetectFaceIds) {\n> + if (faceDetectFaceIds->size() != faceDetectRectangles->size()) {\n> + LOG(HAL, Error) << \"Pipeline returned wrong number of face ids; \"\n> + << \"Expected: \" << faceDetectRectangles->size()\n> + << \", got: \" << faceDetectFaceIds->size();\n> + }\n> + resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, *faceDetectFaceIds);\n> + }\n> +\n> const auto &scalerCrop = metadata.get(controls::ScalerCrop);\n> if (scalerCrop) {\n> const Rectangle &crop = *scalerCrop;\n> --\n> 2.46.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 0F4FBC3257\n\tfor ;\n\tTue, 1 Oct 2024 07:18:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 2E52E6350F;\n\tTue, 1 Oct 2024 09:18:12 +0200 (CEST)","from mail-lj1-x229.google.com (mail-lj1-x229.google.com\n\t[IPv6:2a00:1450:4864:20::229])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9B4A06350E\n\tfor ;\n\tTue, 1 Oct 2024 09:18:10 +0200 (CEST)","by mail-lj1-x229.google.com with SMTP id\n\t38308e7fff4ca-2f75c56f16aso61108551fa.0\n\tfor ;\n\tTue, 01 Oct 2024 00:18:10 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"d1Lc6AdE\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1727767090; x=1728371890;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=3EDPBY+Bgxd8Tr4bKSpys/oHe1E6ilrjAbjOi25D8AM=;\n\tb=d1Lc6AdEZCHPiMz6frQOI9Jqvwlr8U/JmUCKCwgxH3Yr9V9w/A5Lcfap0sTbj/I6wj\n\tKfrGeoajMG4fcAUAbBRcqGaInD4wYx3bfteX9xuWu4puyTKb5yivn19P7cGhDlV85u1d\n\tHvnOvKlYXnLlGE0GHo3wgxs/Rzc8OdlwdI7UE=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1727767090; x=1728371890;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=3EDPBY+Bgxd8Tr4bKSpys/oHe1E6ilrjAbjOi25D8AM=;\n\tb=RZW5oGQqrktt/NqikjDODCuZPs9wgcno7HG6TkoEMxcCDobRSLwe4JOAA5xNqANtxM\n\tvMh41y1RPWWear11pso5JR2fwDi58sSuOxTPPXTUV+ZomxOgBBjZDcwnGssPRk5A1e5G\n\tHnyd4N55sDgbYXb0+zaIBUlyCXaq2GX2gmB9Ih05UrF/DGoPC/spEKSRXulPlSpqQp8Q\n\tSEXmuk0rwTcXHfkjzn9IjnXcFFhcM2eyaSNAhI8fjH98k0ikYMd4e1wyQpnHs56DD8Yo\n\t4G7ERpzV2zq7wwfwn3dcy9tK2th+xFZrx+FN6XHh+C/vqkh1xqL2tlxApEHOyV4h8A/9\n\tpFow==","X-Gm-Message-State":"AOJu0YwDhCsUkHoHiwKuzFTneiafdpgNNJO8h+X+w/cNJGMavqAKJP6l\n\twmUvgjxEC6ldRBKE6o1AjK0U8wpMY8TwvwgbWaR4UAnUXwnkvoVoAE6ivOEW4LvJ/A324TOhTRw\n\tqGXLn0BpryxpC9jaLwYP349NBYhe8C8GycSTM","X-Google-Smtp-Source":"AGHT+IHQVBkO7VeIWi1K8016iCtF2ynTiPmE29SXyq2uH+hSizAEpO5JqstwfOkaFCB0Q1bHtC4iEcZ+YvG9doDAUUI=","X-Received":"by 2002:a2e:a553:0:b0:2f7:4e8c:9bfa with SMTP id\n\t38308e7fff4ca-2f9d417b47bmr78488881fa.33.1727767089556;\n\tTue, 01 Oct 2024 00:18:09 -0700 (PDT)","MIME-Version":"1.0","References":"<20240930195915.152187-1-jacopo.mondi@ideasonboard.com>\n\t<20240930195915.152187-6-jacopo.mondi@ideasonboard.com>","In-Reply-To":"<20240930195915.152187-6-jacopo.mondi@ideasonboard.com>","From":"Cheng-Hao Yang ","Date":"Tue, 1 Oct 2024 15:17:58 +0800","Message-ID":"","Subject":"Re: [PATCH v9 5/5] libcamera: android: Add face detection control\n\tsupport","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org,\n\tHarvey Yang ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]