[{"id":32439,"web_url":"https://patchwork.libcamera.org/comment/32439/","msgid":"","date":"2024-11-28T15:35:26","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> From: Han-Lin Chen \n>\n> With partial result, some metadata, which needs to be added into Exif,\n> may be sent back to framework earlier before Jpeg post-processing.\n> Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> so Jpeg post-processing doesn't need to reference to current metadata.\n>\n> Signed-off-by: Han-Lin Chen \n> Co-developed-by: Harvey Yang \n> Signed-off-by: Harvey Yang \n> ---\n> src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> src/android/camera_device.h | 2 ++\n> src/android/camera_request.h | 6 ++++++\n> src/android/camera_stream.h | 4 ++++\n> src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> 5 files changed, 45 insertions(+), 5 deletions(-)\n>\n> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> index 9fd851bc8..e085e18b2 100644\n> --- a/src/android/camera_device.cpp\n> +++ b/src/android/camera_device.cpp\n> @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> \t\tCameraStream *stream = iter->first;\n> \t\tStreamBuffer *buffer = iter->second;\n>\n> +\t\tif (stream->isJpegStream()) {\n> +\t\t\tgenerateJpegExifMetadata(descriptor, buffer);\n> +\t\t}\n> +\n\nno {} for single line statements\n\n> \t\tFrameBuffer *src = request->findBuffer(stream->stream());\n> \t\tif (!src) {\n> \t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> \tcallbacks_->notify(callbacks_, ¬ify);\n> }\n>\n> +/*\n> + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> + */\n> +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> +\t\t\t\t\t StreamBuffer *buffer) const\n> +{\n> +\tconst ControlList &metadata = request->request_->metadata();\n> +\tauto &jpegExifMetadata = buffer->jpegExifMetadata;\n> +\tjpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> +\n> +\tconst int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> +\tjpegExifMetadata->sensorExposureTime = exposureTime;\n> +\n> +\t/*\n> +\t * todo: Android Sensitivity should only include analog gain X digital\n\n\\todo\n\n> +\t * gain from sensor. Digital gain on ISP shouldn't be included.\n\nmmm, I guess how the gain is split between analogue and digital on the\nsensor is up to the IPA implementation, and currently I only see vc4\nhandling it and it sets it on the ISP.\n\nI wonder if you couldn't simply use AnalogueGain here\n\n> +\t * Calculate sensitivity accordingly when we can differentiate\n> +\t * the source of digital gains.\n> +\t */\n> +\tjpegExifMetadata->sensorSensitivityISO = 100;\n> +}\n> +\n> /*\n> * Produce a set of fixed result metadata.\n> */\n> diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> index 815a695d1..3c46ff918 100644\n> --- a/src/android/camera_device.h\n> +++ b/src/android/camera_device.h\n> @@ -102,6 +102,8 @@ private:\n> \tvoid sendCaptureResult(Camera3RequestDescriptor *request) const;\n> \tvoid setBufferStatus(StreamBuffer &buffer,\n> \t\t\t StreamBuffer::Status status);\n> +\tvoid generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> +\t\t\t\t StreamBuffer *buffer) const;\n> \tstd::unique_ptr getResultMetadata(\n> \t\tconst Camera3RequestDescriptor &descriptor) const;\n>\n> diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> index bd75d4595..bd87b36fd 100644\n> --- a/src/android/camera_request.h\n> +++ b/src/android/camera_request.h\n> @@ -44,6 +44,11 @@ public:\n> \tStreamBuffer(StreamBuffer &&);\n> \tStreamBuffer &operator=(StreamBuffer &&);\n>\n> +\tstruct JpegExifMetadata {\n> +\t\tint64_t sensorExposureTime;\n> +\t\tint32_t sensorSensitivityISO;\n> +\t};\n> +\n> \tCameraStream *stream;\n> \tbuffer_handle_t *camera3Buffer;\n> \tstd::unique_ptr frameBuffer;\n> @@ -51,6 +56,7 @@ public:\n> \tStatus status = Status::Success;\n> \tconst libcamera::FrameBuffer *srcBuffer = nullptr;\n> \tstd::unique_ptr dstBuffer;\n> +\tstd::optional jpegExifMetadata;\n> \tCamera3RequestDescriptor *request;\n>\n> private:\n> diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> index 30f64f690..47cd7ab85 100644\n> --- a/src/android/camera_stream.h\n> +++ b/src/android/camera_stream.h\n> @@ -125,6 +125,10 @@ public:\n> \tconst libcamera::StreamConfiguration &configuration() const;\n> \tlibcamera::Stream *stream() const;\n> \tCameraStream *sourceStream() const { return sourceStream_; }\n> +\tbool isJpegStream() const\n> +\t{\n> +\t\treturn camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> +\t}\n>\n> \tint configure();\n> \tint process(StreamBuffer *streamBuffer);\n> diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> index f5a90785d..48782b574 100644\n> --- a/src/android/jpeg/post_processor_jpeg.cpp\n> +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n>\n> \tconst FrameBuffer &source = *streamBuffer->srcBuffer;\n> \tCameraBuffer *destination = streamBuffer->dstBuffer.get();\n> +\tconst std::optional &jpegExifMetadata =\n> +\t\tstreamBuffer->jpegExifMetadata;\n>\n> \tASSERT(destination->numPlanes() == 1);\n> +\tASSERT(jpegExifMetadata.has_value());\n\nThis means it's not optional, isn't it ?\n\n>\n> \tconst CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> \tCameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> \t */\n> \texif.setTimestamp(std::time(nullptr), 0ms);\n>\n> -\tret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> -\texif.setExposureTime(ret ? *entry.data.i64 : 0);\n> +\t/* Exif requires nsec for exposure time */\n> +\texif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> +\texif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> +\n\nStreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\nto. From there you could get the Request metadata as you currently do\nin CameraDevice::generateJpegExifMetadata().\n\nWhat is the advantage of caching the jpegExifMetadata at\nCameraDevice::requestComplete() time ?\n\nThanks\n j\n\n> \tret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> \tif (ret)\n> \t\texif.setAperture(*entry.data.f);\n>\n> -\tret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> -\texif.setISO(ret ? *entry.data.i32 : 100);\n> -\n> \texif.setFlash(Exif::Flash::FlashNotPresent);\n> \texif.setWhiteBalance(Exif::WhiteBalance::Auto);\n>\n> --\n> 2.47.0.338.g60cca15819-goog\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id C5A95BDE6B\n\tfor ;\n\tThu, 28 Nov 2024 15:35:32 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id EB72D65FBE;\n\tThu, 28 Nov 2024 16:35:31 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2757A65898\n\tfor ;\n\tThu, 28 Nov 2024 16:35:30 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 1630F526;\n\tThu, 28 Nov 2024 16:35:06 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"EzqOjJ3o\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1732808106;\n\tbh=he6ARZFPn3W4DGW0TDm9GCx0+p7OHeMFVy7Pe4wuX2E=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=EzqOjJ3o/YJpdBuIHY/fgSJjKee1XULCrReeYDRu9imU/MP5csWKEVIqtpXQ1KaXt\n\t8fJPI7F6gSNY8Xhe6nDIp+MRv7vU74InHlwHBvch0FGsmjLBYpfAOgeEOINKzkRFqs\n\t8mN7wE0m0wg97++gLQ782QvHYSeqUg2DYe/vLvZ0=","Date":"Thu, 28 Nov 2024 16:35:26 +0100","From":"Jacopo Mondi ","To":"Harvey Yang ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20241127092632.3145984-9-chenghaoyang@chromium.org>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32456,"web_url":"https://patchwork.libcamera.org/comment/32456/","msgid":"","date":"2024-11-29T09:05:30","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > From: Han-Lin Chen \n> >\n> > With partial result, some metadata, which needs to be added into Exif,\n> > may be sent back to framework earlier before Jpeg post-processing.\n> > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > so Jpeg post-processing doesn't need to reference to current metadata.\n> >\n> > Signed-off-by: Han-Lin Chen \n> > Co-developed-by: Harvey Yang \n> > Signed-off-by: Harvey Yang \n> > ---\n> > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > src/android/camera_device.h | 2 ++\n> > src/android/camera_request.h | 6 ++++++\n> > src/android/camera_stream.h | 4 ++++\n> > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > 5 files changed, 45 insertions(+), 5 deletions(-)\n> >\n> > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > index 9fd851bc8..e085e18b2 100644\n> > --- a/src/android/camera_device.cpp\n> > +++ b/src/android/camera_device.cpp\n> > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > CameraStream *stream = iter->first;\n> > StreamBuffer *buffer = iter->second;\n> >\n> > + if (stream->isJpegStream()) {\n> > + generateJpegExifMetadata(descriptor, buffer);\n> > + }\n> > +\n>\n> no {} for single line statements\n>\n> > FrameBuffer *src = request->findBuffer(stream->stream());\n> > if (!src) {\n> > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > callbacks_->notify(callbacks_, ¬ify);\n> > }\n> >\n> > +/*\n> > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > + */\n> > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > + StreamBuffer *buffer) const\n> > +{\n> > + const ControlList &metadata = request->request_->metadata();\n> > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > +\n> > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > +\n> > + /*\n> > + * todo: Android Sensitivity should only include analog gain X digital\n>\n> \\todo\n>\n> > + * gain from sensor. Digital gain on ISP shouldn't be included.\n>\n> mmm, I guess how the gain is split between analogue and digital on the\n> sensor is up to the IPA implementation, and currently I only see vc4\n> handling it and it sets it on the ISP.\n>\n> I wonder if you couldn't simply use AnalogueGain here\n>\n> > + * Calculate sensitivity accordingly when we can differentiate\n> > + * the source of digital gains.\n> > + */\n> > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > +}\n> > +\n> > /*\n> > * Produce a set of fixed result metadata.\n> > */\n> > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > index 815a695d1..3c46ff918 100644\n> > --- a/src/android/camera_device.h\n> > +++ b/src/android/camera_device.h\n> > @@ -102,6 +102,8 @@ private:\n> > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > void setBufferStatus(StreamBuffer &buffer,\n> > StreamBuffer::Status status);\n> > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > + StreamBuffer *buffer) const;\n> > std::unique_ptr getResultMetadata(\n> > const Camera3RequestDescriptor &descriptor) const;\n> >\n> > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > index bd75d4595..bd87b36fd 100644\n> > --- a/src/android/camera_request.h\n> > +++ b/src/android/camera_request.h\n> > @@ -44,6 +44,11 @@ public:\n> > StreamBuffer(StreamBuffer &&);\n> > StreamBuffer &operator=(StreamBuffer &&);\n> >\n> > + struct JpegExifMetadata {\n> > + int64_t sensorExposureTime;\n> > + int32_t sensorSensitivityISO;\n> > + };\n> > +\n> > CameraStream *stream;\n> > buffer_handle_t *camera3Buffer;\n> > std::unique_ptr frameBuffer;\n> > @@ -51,6 +56,7 @@ public:\n> > Status status = Status::Success;\n> > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > std::unique_ptr dstBuffer;\n> > + std::optional jpegExifMetadata;\n> > Camera3RequestDescriptor *request;\n> >\n> > private:\n> > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > index 30f64f690..47cd7ab85 100644\n> > --- a/src/android/camera_stream.h\n> > +++ b/src/android/camera_stream.h\n> > @@ -125,6 +125,10 @@ public:\n> > const libcamera::StreamConfiguration &configuration() const;\n> > libcamera::Stream *stream() const;\n> > CameraStream *sourceStream() const { return sourceStream_; }\n> > + bool isJpegStream() const\n> > + {\n> > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > + }\n> >\n> > int configure();\n> > int process(StreamBuffer *streamBuffer);\n> > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > index f5a90785d..48782b574 100644\n> > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> >\n> > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > + const std::optional &jpegExifMetadata =\n> > + streamBuffer->jpegExifMetadata;\n> >\n> > ASSERT(destination->numPlanes() == 1);\n> > + ASSERT(jpegExifMetadata.has_value());\n>\n> This means it's not optional, isn't it ?\n>\n> >\n> > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > */\n> > exif.setTimestamp(std::time(nullptr), 0ms);\n> >\n> > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > + /* Exif requires nsec for exposure time */\n> > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > +\n>\n> StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> to. From there you could get the Request metadata as you currently do\n> in CameraDevice::generateJpegExifMetadata().\n\nThat's a very good question, only that I wonder how we can handle the\nthreading issue properly.\n\nThe current implementation of `Request::metadata()` [1] doesn't seem\nto consider race conditions, and our goal is to support partial results,\nwhich means that the post processor thread might try to access metadata\nwhen the request is still being processed in the pipeline handler, which\nmight set further metadata tags.\n\nWDYT?\n\n[1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n\n>\n> What is the advantage of caching the jpegExifMetadata at\n> CameraDevice::requestComplete() time ?\n>\n> Thanks\n> j\n>\n> > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > if (ret)\n> > exif.setAperture(*entry.data.f);\n> >\n> > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > -\n> > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> >\n> > --\n> > 2.47.0.338.g60cca15819-goog\n> >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4BD66C31E9\n\tfor ;\n\tFri, 29 Nov 2024 09:05:47 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7A15F65FFC;\n\tFri, 29 Nov 2024 10:05:46 +0100 (CET)","from mail-lj1-x235.google.com (mail-lj1-x235.google.com\n\t[IPv6:2a00:1450:4864:20::235])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 987E260CE6\n\tfor ;\n\tFri, 29 Nov 2024 10:05:44 +0100 (CET)","by mail-lj1-x235.google.com with SMTP id\n\t38308e7fff4ca-2ffc76368c6so27195771fa.0\n\tfor ;\n\tFri, 29 Nov 2024 01:05:44 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"cZWyv1AR\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1732871144; x=1733475944;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=f4oJXEM7bUpJ5iLjEFFEFNhiHFLtrMaASdY6ACt5lo0=;\n\tb=cZWyv1ARI8i9aBglCUOPIaUjUv3iTawdBWSP4AtHHskgOE318oHiR2zkizPqgBXw9k\n\t6/Z5WqTPhFpH9Lamzvqtl6JPhByRt5YEAwKCV0vpNXNG+OI8J9zfU/AXYomaJhX8vHA6\n\tGKdzLlhsL2WgO/J8QyaPjq2fqkgEakDsVcsos=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1732871144; x=1733475944;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=f4oJXEM7bUpJ5iLjEFFEFNhiHFLtrMaASdY6ACt5lo0=;\n\tb=inUAhAemVpovmoOYqW5gqFIjsqzNtqWCRaQmyRvU1igce2ysjslFPmwcRE+bKhZF72\n\t//N/QDR/PGmqXRuqYHhuUEwJD9YGHQG/N1mFW6HP0ho6LvHcvbTCoH5PUyXLgP9m8qA9\n\tDFNYhuRsS+XyYq0p95WfR3fHVermFTmZWNq5/yb+kGYlZgTvhvaOVZx2BEByTRVZWFPv\n\t0l7vez51t0ZpHrl9Idm7EkKxVGxz7rs6pQ6/6c+ZjoHoJoM7OUrbh0IqiSdLwqnyCgT7\n\tV0OhXdBAxaDj/Sl672G76zizYflfjZQbeE/WQS2hA758cBVLWp/My31p6pjY0s9e3kZ3\n\tzYtA==","X-Gm-Message-State":"AOJu0YwSwH2aMHoa9I6+Z/eHiod/D/3jBYlvIFRO/Rk+0CYkF/5x2P7b\n\t/ZZhf5D2mjyxgZ5nnHKv9N/Ik2jH6ElUbyk2xrhtCSmayNvbYQnBn7lWmuTFQIvjn8uug9bcm2M\n\t5saiid8TTlAN0/gwLd/GjRVA5XiJwzXaDrq3h","X-Gm-Gg":"ASbGncuJ6nmIX8G2kQQsAoQtTwZbf56/5IENa1AWJhjxstPREYTr/y1yMwu0ClNfxRZ\n\t04eszeC5TFNsDRIqavz1JH7kE1qC1Bpk9dgQ5VuXfafI+DMWfxt6GbDlBGl0=","X-Google-Smtp-Source":"AGHT+IGalnbHEZwv5NhoeJQZnS5SW4uESBVa6o8dEjcwmArkn9rkJkCSs6/qO6M36UL4tg505+ysMXiPWNJqwK7XBbA=","X-Received":"by 2002:a2e:bcc2:0:b0:2ff:95d7:9ed2 with SMTP id\n\t38308e7fff4ca-2ffd60c6f1emr101842281fa.32.1732871141974;\n\tFri, 29 Nov 2024 01:05:41 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Fri, 29 Nov 2024 17:05:30 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32457,"web_url":"https://patchwork.libcamera.org/comment/32457/","msgid":"","date":"2024-11-29T09:17:30","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n wrote:\n>\n> Hi Jacopo,\n>\n> On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > From: Han-Lin Chen \n> > >\n> > > With partial result, some metadata, which needs to be added into Exif,\n> > > may be sent back to framework earlier before Jpeg post-processing.\n> > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > >\n> > > Signed-off-by: Han-Lin Chen \n> > > Co-developed-by: Harvey Yang \n> > > Signed-off-by: Harvey Yang \n> > > ---\n> > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > src/android/camera_device.h | 2 ++\n> > > src/android/camera_request.h | 6 ++++++\n> > > src/android/camera_stream.h | 4 ++++\n> > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > >\n> > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > index 9fd851bc8..e085e18b2 100644\n> > > --- a/src/android/camera_device.cpp\n> > > +++ b/src/android/camera_device.cpp\n> > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > CameraStream *stream = iter->first;\n> > > StreamBuffer *buffer = iter->second;\n> > >\n> > > + if (stream->isJpegStream()) {\n> > > + generateJpegExifMetadata(descriptor, buffer);\n> > > + }\n> > > +\n> >\n> > no {} for single line statements\n\nDone\n\n> >\n> > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > if (!src) {\n> > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > callbacks_->notify(callbacks_, ¬ify);\n> > > }\n> > >\n> > > +/*\n> > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > + */\n> > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > + StreamBuffer *buffer) const\n> > > +{\n> > > + const ControlList &metadata = request->request_->metadata();\n> > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > +\n> > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > +\n> > > + /*\n> > > + * todo: Android Sensitivity should only include analog gain X digital\n> >\n> > \\todo\n\ndone\n\n> >\n> > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> >\n> > mmm, I guess how the gain is split between analogue and digital on the\n> > sensor is up to the IPA implementation, and currently I only see vc4\n> > handling it and it sets it on the ISP.\n> >\n> > I wonder if you couldn't simply use AnalogueGain here\n\nI think the comment here is assuming the further changes that use\nAnalogueGain directly here, while might not be needed in this patch...\nRemoved.\n\nI'll remember (hopefully) when we use AnalogueGain here in the\nfollowing patches.\n\n> >\n> > > + * Calculate sensitivity accordingly when we can differentiate\n> > > + * the source of digital gains.\n> > > + */\n> > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > +}\n> > > +\n> > > /*\n> > > * Produce a set of fixed result metadata.\n> > > */\n> > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > index 815a695d1..3c46ff918 100644\n> > > --- a/src/android/camera_device.h\n> > > +++ b/src/android/camera_device.h\n> > > @@ -102,6 +102,8 @@ private:\n> > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > void setBufferStatus(StreamBuffer &buffer,\n> > > StreamBuffer::Status status);\n> > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > + StreamBuffer *buffer) const;\n> > > std::unique_ptr getResultMetadata(\n> > > const Camera3RequestDescriptor &descriptor) const;\n> > >\n> > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > index bd75d4595..bd87b36fd 100644\n> > > --- a/src/android/camera_request.h\n> > > +++ b/src/android/camera_request.h\n> > > @@ -44,6 +44,11 @@ public:\n> > > StreamBuffer(StreamBuffer &&);\n> > > StreamBuffer &operator=(StreamBuffer &&);\n> > >\n> > > + struct JpegExifMetadata {\n> > > + int64_t sensorExposureTime;\n> > > + int32_t sensorSensitivityISO;\n> > > + };\n> > > +\n> > > CameraStream *stream;\n> > > buffer_handle_t *camera3Buffer;\n> > > std::unique_ptr frameBuffer;\n> > > @@ -51,6 +56,7 @@ public:\n> > > Status status = Status::Success;\n> > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > std::unique_ptr dstBuffer;\n> > > + std::optional jpegExifMetadata;\n> > > Camera3RequestDescriptor *request;\n> > >\n> > > private:\n> > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > index 30f64f690..47cd7ab85 100644\n> > > --- a/src/android/camera_stream.h\n> > > +++ b/src/android/camera_stream.h\n> > > @@ -125,6 +125,10 @@ public:\n> > > const libcamera::StreamConfiguration &configuration() const;\n> > > libcamera::Stream *stream() const;\n> > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > + bool isJpegStream() const\n> > > + {\n> > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > + }\n> > >\n> > > int configure();\n> > > int process(StreamBuffer *streamBuffer);\n> > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > index f5a90785d..48782b574 100644\n> > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > >\n> > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > + const std::optional &jpegExifMetadata =\n> > > + streamBuffer->jpegExifMetadata;\n> > >\n> > > ASSERT(destination->numPlanes() == 1);\n> > > + ASSERT(jpegExifMetadata.has_value());\n> >\n> > This means it's not optional, isn't it ?\n\nMeans it's not std::nullopt. Any suggestions?\n\nBR,\nHarvey\n\n> >\n> > >\n> > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > */\n> > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > >\n> > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > + /* Exif requires nsec for exposure time */\n> > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > +\n> >\n> > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > to. From there you could get the Request metadata as you currently do\n> > in CameraDevice::generateJpegExifMetadata().\n>\n> That's a very good question, only that I wonder how we can handle the\n> threading issue properly.\n>\n> The current implementation of `Request::metadata()` [1] doesn't seem\n> to consider race conditions, and our goal is to support partial results,\n> which means that the post processor thread might try to access metadata\n> when the request is still being processed in the pipeline handler, which\n> might set further metadata tags.\n>\n> WDYT?\n>\n> [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n>\n> >\n> > What is the advantage of caching the jpegExifMetadata at\n> > CameraDevice::requestComplete() time ?\n> >\n> > Thanks\n> > j\n> >\n> > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > if (ret)\n> > > exif.setAperture(*entry.data.f);\n> > >\n> > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > -\n> > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > >\n> > > --\n> > > 2.47.0.338.g60cca15819-goog\n> > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6A086C3220\n\tfor ;\n\tFri, 29 Nov 2024 09:17:46 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id BF8176600A;\n\tFri, 29 Nov 2024 10:17:45 +0100 (CET)","from mail-lj1-x22e.google.com (mail-lj1-x22e.google.com\n\t[IPv6:2a00:1450:4864:20::22e])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 8AC0A65F97\n\tfor ;\n\tFri, 29 Nov 2024 10:17:43 +0100 (CET)","by mail-lj1-x22e.google.com with SMTP id\n\t38308e7fff4ca-2ffdd9fc913so17386941fa.3\n\tfor ;\n\tFri, 29 Nov 2024 01:17:43 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"lT3J0XE1\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1732871863; x=1733476663;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=9hAUwF1tdxDMQ6oOaSPvliqq8lSfBB+/tYCKda9KP8I=;\n\tb=lT3J0XE18IZusJtaN+VqApXsR8O7oq9FgpKDtxxyiomUPLqavxt4MOLhjCnqpHW7go\n\txNGAQbVMK/YRAby6tRHboq2DSDJYlIY/rg4I0Adp41qO7hQa45zsiKpaBh8Bu6HjgI67\n\tn9XQ3vU0E3i0YAQtcqVFc0JIK/o60YapEzqT0=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1732871863; x=1733476663;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=9hAUwF1tdxDMQ6oOaSPvliqq8lSfBB+/tYCKda9KP8I=;\n\tb=fTamoWlhmnwhi7S79ZNc/pNdhQ56rnRdV4qpI2UayEFp9zvUpPmtwohp7/yLDhW1N9\n\tgCDejl2lMHz1NS99lIMp4Z9DndTqTblDwTZn16FeQLI12z5t4QrKtysX/9FiGj9kxsEv\n\tgdUt/+dzd8pjgRLKQ4KjvKxwajRrMhMusdp43g4okwJexwwv/KH7ReRoQnI8vxj6swRH\n\t2r+orrBg3iEOIJkVLUaoU8+vhEuZNrnoOWNsI/J/XnwRRH7EoDCVzpkb/zORLwZNLlOd\n\teEdPrwM7WPQwJ71cGYcsZLY7FxbVB3+WtbglsF+ioTSxCA9muij1T3VT3uYWVq4CE0Bl\n\tviJw==","X-Gm-Message-State":"AOJu0Yxfq3PDbFSShg7yzrSIgaMTF+hjhwSYNNjk+Dyf/GBBzE1MYUch\n\toE+ll87BtHXb9HjfUUph9fmps5a+Bj4f36lmKmjKoevl1uajaBmA8IDJ4OuTceiGdh9iozkkZK0\n\tHKTOLL1IeHozU99gkX3k6Ec8g3EyQhEO8vbKg","X-Gm-Gg":"ASbGncthb9BomLNO7r5ltAh0JotvbfKZUjRuCT5Sxtbc2EWt44HcFrVWcjhGkqOR5yn\n\tg7WymPnwWSnob8trE0WKBCHmzKqoIlGPYR1L3mx4M5Ann0d/SXFKHEq+fnlI=","X-Google-Smtp-Source":"AGHT+IEWAc3qKWC5FoA3a8et9ZJwdHuMbcXJ5HGqJ+dvqPX6QycKv59XdAS4D1RPA/5RJiO7u4fycS0rqP4tNP4/dAQ=","X-Received":"by 2002:a2e:bea3:0:b0:2ff:b812:36e6 with SMTP id\n\t38308e7fff4ca-2ffd60e2197mr61545171fa.35.1732871862647;\n\tFri, 29 Nov 2024 01:17:42 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Fri, 29 Nov 2024 17:17:30 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32487,"web_url":"https://patchwork.libcamera.org/comment/32487/","msgid":"","date":"2024-12-02T17:04:10","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> wrote:\n> >\n> > Hi Jacopo,\n> >\n> > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > From: Han-Lin Chen \n> > > >\n> > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > >\n> > > > Signed-off-by: Han-Lin Chen \n> > > > Co-developed-by: Harvey Yang \n> > > > Signed-off-by: Harvey Yang \n> > > > ---\n> > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > src/android/camera_device.h | 2 ++\n> > > > src/android/camera_request.h | 6 ++++++\n> > > > src/android/camera_stream.h | 4 ++++\n> > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > >\n> > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > index 9fd851bc8..e085e18b2 100644\n> > > > --- a/src/android/camera_device.cpp\n> > > > +++ b/src/android/camera_device.cpp\n> > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > CameraStream *stream = iter->first;\n> > > > StreamBuffer *buffer = iter->second;\n> > > >\n> > > > + if (stream->isJpegStream()) {\n> > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > + }\n> > > > +\n> > >\n> > > no {} for single line statements\n>\n> Done\n>\n> > >\n> > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > if (!src) {\n> > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > }\n> > > >\n> > > > +/*\n> > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > + */\n> > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > + StreamBuffer *buffer) const\n> > > > +{\n> > > > + const ControlList &metadata = request->request_->metadata();\n> > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > +\n> > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > +\n> > > > + /*\n> > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > >\n> > > \\todo\n>\n> done\n>\n> > >\n> > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > >\n> > > mmm, I guess how the gain is split between analogue and digital on the\n> > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > handling it and it sets it on the ISP.\n> > >\n> > > I wonder if you couldn't simply use AnalogueGain here\n>\n> I think the comment here is assuming the further changes that use\n> AnalogueGain directly here, while might not be needed in this patch...\n> Removed.\n\nI'm not sure I get this comment in full. I was suggesting to use\nAnalogueGain directly here, but if it isn't required, I'm fine with\nkeeping a \\todo\n\n>\n> I'll remember (hopefully) when we use AnalogueGain here in the\n> following patches.\n>\n> > >\n> > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > + * the source of digital gains.\n> > > > + */\n> > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > +}\n> > > > +\n> > > > /*\n> > > > * Produce a set of fixed result metadata.\n> > > > */\n> > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > index 815a695d1..3c46ff918 100644\n> > > > --- a/src/android/camera_device.h\n> > > > +++ b/src/android/camera_device.h\n> > > > @@ -102,6 +102,8 @@ private:\n> > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > StreamBuffer::Status status);\n> > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > + StreamBuffer *buffer) const;\n> > > > std::unique_ptr getResultMetadata(\n> > > > const Camera3RequestDescriptor &descriptor) const;\n> > > >\n> > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > index bd75d4595..bd87b36fd 100644\n> > > > --- a/src/android/camera_request.h\n> > > > +++ b/src/android/camera_request.h\n> > > > @@ -44,6 +44,11 @@ public:\n> > > > StreamBuffer(StreamBuffer &&);\n> > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > >\n> > > > + struct JpegExifMetadata {\n> > > > + int64_t sensorExposureTime;\n> > > > + int32_t sensorSensitivityISO;\n> > > > + };\n> > > > +\n> > > > CameraStream *stream;\n> > > > buffer_handle_t *camera3Buffer;\n> > > > std::unique_ptr frameBuffer;\n> > > > @@ -51,6 +56,7 @@ public:\n> > > > Status status = Status::Success;\n> > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > std::unique_ptr dstBuffer;\n> > > > + std::optional jpegExifMetadata;\n> > > > Camera3RequestDescriptor *request;\n> > > >\n> > > > private:\n> > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > index 30f64f690..47cd7ab85 100644\n> > > > --- a/src/android/camera_stream.h\n> > > > +++ b/src/android/camera_stream.h\n> > > > @@ -125,6 +125,10 @@ public:\n> > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > libcamera::Stream *stream() const;\n> > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > + bool isJpegStream() const\n> > > > + {\n> > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > + }\n> > > >\n> > > > int configure();\n> > > > int process(StreamBuffer *streamBuffer);\n> > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > index f5a90785d..48782b574 100644\n> > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > >\n> > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > + const std::optional &jpegExifMetadata =\n> > > > + streamBuffer->jpegExifMetadata;\n> > > >\n> > > > ASSERT(destination->numPlanes() == 1);\n> > > > + ASSERT(jpegExifMetadata.has_value());\n> > >\n> > > This means it's not optional, isn't it ?\n>\n> Means it's not std::nullopt. Any suggestions?\n\nyeah, what I meant was \"if you ASSERT()\" you expect it to be always\npopulated, so std::optional<> doesn't bring any value. But you\nprobably want to make sure that jpegExifMetadata has been populated\nwhen this function is called, so feel free to keep std::optional<>\n\n>\n> BR,\n> Harvey\n>\n> > >\n> > > >\n> > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > */\n> > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > >\n> > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > + /* Exif requires nsec for exposure time */\n> > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > +\n> > >\n> > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > to. From there you could get the Request metadata as you currently do\n> > > in CameraDevice::generateJpegExifMetadata().\n> >\n> > That's a very good question, only that I wonder how we can handle the\n> > threading issue properly.\n> >\n> > The current implementation of `Request::metadata()` [1] doesn't seem\n> > to consider race conditions, and our goal is to support partial results,\n> > which means that the post processor thread might try to access metadata\n> > when the request is still being processed in the pipeline handler, which\n> > might set further metadata tags.\n\nIs this an issue about threading or about the order in which metadata\nand buffer completes ?\n\nI see in the next patch you call generateJpegExifMetadata() on a\n((Mapped|Internal) && Jpeg) bufferComplete event.\n\nhttps://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n\nAnd immediately after, if such a buffer has been completed, you call\nprocess() on it\nhttps://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n\nIs this right ?\n\nSo when the buffer completes, you inspect the so-far-completed\nmetadata and extract ExposureTime (and AnalogueGain eventually).\n\n1) Why you don't do that at metadataAvailable time ?\n2) What does guarantee that the pipeline has populated ExposureTime at\nthe time the buffer to process has completed ?\n\n\n\n\n> >\n> > WDYT?\n> >\n> > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> >\n\nI might have missed why this is related :)\n\n> > >\n> > > What is the advantage of caching the jpegExifMetadata at\n> > > CameraDevice::requestComplete() time ?\n> > >\n> > > Thanks\n> > > j\n> > >\n> > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > if (ret)\n> > > > exif.setAperture(*entry.data.f);\n> > > >\n> > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > -\n> > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > >\n> > > > --\n> > > > 2.47.0.338.g60cca15819-goog\n> > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 240E0BDCBF\n\tfor ;\n\tMon, 2 Dec 2024 17:04:19 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 67F2E66072;\n\tMon, 2 Dec 2024 18:04:18 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CC6876605F\n\tfor ;\n\tMon, 2 Dec 2024 18:04:16 +0100 (CET)","from ideasonboard.com (mob-5-90-236-68.net.vodafone.it\n\t[5.90.236.68])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A5F3E514;\n\tMon, 2 Dec 2024 18:03:49 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"SvIw7U9+\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1733159030;\n\tbh=bivwOw4ObyFxDtROLQaHHgAatOWuPzWhUt91m1jMEEU=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=SvIw7U9+IcyPVimRK9Zznh+05J6UxYMG/+mmIxNEeAic4OGi/o8gV+vooscxC1ADy\n\tUDP8oMYbW1Rl+zqO1j98W9KjnJ4P59rYvgi9U9FKr3uKivX34SqEf9FQ+B5qK9iPHV\n\tuTVEVlwf+S7FmGfLvgTi2W5NdQxdo71t1EqtSTBw=","Date":"Mon, 2 Dec 2024 18:04:10 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32499,"web_url":"https://patchwork.libcamera.org/comment/32499/","msgid":"","date":"2024-12-03T14:33:22","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > wrote:\n> > >\n> > > Hi Jacopo,\n> > >\n> > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > From: Han-Lin Chen \n> > > > >\n> > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > >\n> > > > > Signed-off-by: Han-Lin Chen \n> > > > > Co-developed-by: Harvey Yang \n> > > > > Signed-off-by: Harvey Yang \n> > > > > ---\n> > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > src/android/camera_device.h | 2 ++\n> > > > > src/android/camera_request.h | 6 ++++++\n> > > > > src/android/camera_stream.h | 4 ++++\n> > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > >\n> > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > --- a/src/android/camera_device.cpp\n> > > > > +++ b/src/android/camera_device.cpp\n> > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > CameraStream *stream = iter->first;\n> > > > > StreamBuffer *buffer = iter->second;\n> > > > >\n> > > > > + if (stream->isJpegStream()) {\n> > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > + }\n> > > > > +\n> > > >\n> > > > no {} for single line statements\n> >\n> > Done\n> >\n> > > >\n> > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > if (!src) {\n> > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > }\n> > > > >\n> > > > > +/*\n> > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > + */\n> > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > + StreamBuffer *buffer) const\n> > > > > +{\n> > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > +\n> > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > +\n> > > > > + /*\n> > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > >\n> > > > \\todo\n> >\n> > done\n> >\n> > > >\n> > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > >\n> > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > handling it and it sets it on the ISP.\n> > > >\n> > > > I wonder if you couldn't simply use AnalogueGain here\n> >\n> > I think the comment here is assuming the further changes that use\n> > AnalogueGain directly here, while might not be needed in this patch...\n> > Removed.\n>\n> I'm not sure I get this comment in full. I was suggesting to use\n> AnalogueGain directly here, but if it isn't required, I'm fine with\n> keeping a \\todo\n\nYeah I know, while I think using AnalogueGain or even a \\todo is\nirrelevant in this patch. We may add a separate one to use\nAnalogueGain directly :)\n\n>\n> >\n> > I'll remember (hopefully) when we use AnalogueGain here in the\n> > following patches.\n> >\n> > > >\n> > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > + * the source of digital gains.\n> > > > > + */\n> > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > +}\n> > > > > +\n> > > > > /*\n> > > > > * Produce a set of fixed result metadata.\n> > > > > */\n> > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > index 815a695d1..3c46ff918 100644\n> > > > > --- a/src/android/camera_device.h\n> > > > > +++ b/src/android/camera_device.h\n> > > > > @@ -102,6 +102,8 @@ private:\n> > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > StreamBuffer::Status status);\n> > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > + StreamBuffer *buffer) const;\n> > > > > std::unique_ptr getResultMetadata(\n> > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > >\n> > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > index bd75d4595..bd87b36fd 100644\n> > > > > --- a/src/android/camera_request.h\n> > > > > +++ b/src/android/camera_request.h\n> > > > > @@ -44,6 +44,11 @@ public:\n> > > > > StreamBuffer(StreamBuffer &&);\n> > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > >\n> > > > > + struct JpegExifMetadata {\n> > > > > + int64_t sensorExposureTime;\n> > > > > + int32_t sensorSensitivityISO;\n> > > > > + };\n> > > > > +\n> > > > > CameraStream *stream;\n> > > > > buffer_handle_t *camera3Buffer;\n> > > > > std::unique_ptr frameBuffer;\n> > > > > @@ -51,6 +56,7 @@ public:\n> > > > > Status status = Status::Success;\n> > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > std::unique_ptr dstBuffer;\n> > > > > + std::optional jpegExifMetadata;\n> > > > > Camera3RequestDescriptor *request;\n> > > > >\n> > > > > private:\n> > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > index 30f64f690..47cd7ab85 100644\n> > > > > --- a/src/android/camera_stream.h\n> > > > > +++ b/src/android/camera_stream.h\n> > > > > @@ -125,6 +125,10 @@ public:\n> > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > libcamera::Stream *stream() const;\n> > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > + bool isJpegStream() const\n> > > > > + {\n> > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > + }\n> > > > >\n> > > > > int configure();\n> > > > > int process(StreamBuffer *streamBuffer);\n> > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > index f5a90785d..48782b574 100644\n> > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > >\n> > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > + const std::optional &jpegExifMetadata =\n> > > > > + streamBuffer->jpegExifMetadata;\n> > > > >\n> > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > >\n> > > > This means it's not optional, isn't it ?\n> >\n> > Means it's not std::nullopt. Any suggestions?\n>\n> yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> populated, so std::optional<> doesn't bring any value. But you\n> probably want to make sure that jpegExifMetadata has been populated\n> when this function is called, so feel free to keep std::optional<>\n\nSure :)\n\n>\n> >\n> > BR,\n> > Harvey\n> >\n> > > >\n> > > > >\n> > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > */\n> > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > >\n> > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > + /* Exif requires nsec for exposure time */\n> > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > +\n> > > >\n> > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > to. From there you could get the Request metadata as you currently do\n> > > > in CameraDevice::generateJpegExifMetadata().\n> > >\n> > > That's a very good question, only that I wonder how we can handle the\n> > > threading issue properly.\n> > >\n> > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > to consider race conditions, and our goal is to support partial results,\n> > > which means that the post processor thread might try to access metadata\n> > > when the request is still being processed in the pipeline handler, which\n> > > might set further metadata tags.\n>\n> Is this an issue about threading or about the order in which metadata\n> and buffer completes ?\n\nWhat I meant is the threading issue, while the order is also an issue, true.\n\n>\n> I see in the next patch you call generateJpegExifMetadata() on a\n> ((Mapped|Internal) && Jpeg) bufferComplete event.\n>\n> https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n>\n> And immediately after, if such a buffer has been completed, you call\n> process() on it\n> https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n>\n> Is this right ?\n>\n> So when the buffer completes, you inspect the so-far-completed\n> metadata and extract ExposureTime (and AnalogueGain eventually).\n>\n> 1) Why you don't do that at metadataAvailable time ?\n\nMakes a lot of sense. Will try to apply in the next patch.\n\n> 2) What does guarantee that the pipeline has populated ExposureTime at\n> the time the buffer to process has completed ?\n\nThat's a very good question. I'd like your input: Do you think we\nshould pause the jpeg process until all necessary metadata tags are\navailable?\n\n>\n>\n>\n>\n> > >\n> > > WDYT?\n> > >\n> > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > >\n>\n> I might have missed why this is related :)\n\nOkay, I think I misunderstood the \\todo, but the point is that it\ndoesn't handle the threading issue. We should avoid reading it in the\npost processor thread, especially with partial results, as the Request\nmight not have been completed.\n\nBR,\nHarvey\n\n>\n> > > >\n> > > > What is the advantage of caching the jpegExifMetadata at\n> > > > CameraDevice::requestComplete() time ?\n> > > >\n> > > > Thanks\n> > > > j\n> > > >\n> > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > if (ret)\n> > > > > exif.setAperture(*entry.data.f);\n> > > > >\n> > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > -\n> > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > >\n> > > > > --\n> > > > > 2.47.0.338.g60cca15819-goog\n> > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 40D30BDC71\n\tfor ;\n\tTue, 3 Dec 2024 14:33:38 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id F239C6608F;\n\tTue, 3 Dec 2024 15:33:36 +0100 (CET)","from mail-wr1-x42c.google.com (mail-wr1-x42c.google.com\n\t[IPv6:2a00:1450:4864:20::42c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 6FA2265FD6\n\tfor ;\n\tTue, 3 Dec 2024 15:33:35 +0100 (CET)","by mail-wr1-x42c.google.com with SMTP id\n\tffacd0b85a97d-385f06d0c8eso1608573f8f.0\n\tfor ;\n\tTue, 03 Dec 2024 06:33:35 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"DAZsBBOM\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1733236415; x=1733841215;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=uOc0Gh2CLkKor6Z8DjZ9TsnjZt3v0bsVpyU3xGDIhXU=;\n\tb=DAZsBBOMy3692yu6EZbHZo3RGRLqqRc8tFENRbprS6/6uWdVJE4pn86nU64AHKcNiv\n\ty9O2CpHuRxE0cxVqkV4mhsk0lv13jSM3/HYI/G6dwt91uC13HQD4TudN8t+1KVBT91xe\n\t1URmBGkcHiUjaHC7mOMtrqKJxLO05v0bKLERw=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1733236415; x=1733841215;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=uOc0Gh2CLkKor6Z8DjZ9TsnjZt3v0bsVpyU3xGDIhXU=;\n\tb=EsYY0583BMPAjBifzLFzRDj1z8uW0yvM2o0r5Lwd10LpkkoUFGFlAoqotCLt7cnqFZ\n\tp135Jpzai740MH0Rm3cjGBIctISOTZOmJT3RSqMvsJgvxRPMAolJQyPuxbbzEICjg8OL\n\tsj4uoXLlEgZ9l8fHsyncfO9TtnGlUORBFHgkI64Iny/8fGEFtf/vFDH3UUw1os7KGp0R\n\tDAsY0cghF2EYzgUOKvqLAhF6Kezy+6OhUs4aze/wkO7MJbPC0TJbHv/wdPLOt3+S3XPE\n\tBLUuU37qIpTTOknRCXwp0oNaFIjqec+us5WAZBMFM/v7htSiOcKjuhkIJikQespArKAx\n\tEoSQ==","X-Gm-Message-State":"AOJu0YyAoTOfgpL+2df6Idj2dHA2PJPYi0AdwUary3wyJ08RLRt2MJi3\n\tVdURqa2jflcZNSW2au/XODuEc9if/eQbA5uBfx0NYKiOPcUVtHjviSmy+jLmb7y/YnNG2JYuCVI\n\tYP+/l0/JUiDbGHkhZp7T7F+y2dshb30nCblq2","X-Gm-Gg":"ASbGncusoIqNiA33zPp9SDs8JjpduT74ARLfwzGZ5ljPe/UrTyudY4saC+NF2Nptl8n\n\t+N6H1ulZfhOTT9dG/KThI0CZ29QtWEzegGFdqVtlInf02cWh2V+Iq0BMbIzoQ13kmMw==","X-Google-Smtp-Source":"AGHT+IEXVKm3Lti8G2krP4K6w+yAK5suZsh8ZAVtEvQwrK4muszwlehwhxKLPTjUGmitYudGSgdrt5xOxJ0E4bQy5NM=","X-Received":"by 2002:a05:6000:184c:b0:37d:5364:d738 with SMTP id\n\tffacd0b85a97d-385fd429b21mr2364893f8f.45.1733236414680;\n\tTue, 03 Dec 2024 06:33:34 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Tue, 3 Dec 2024 22:33:22 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32506,"web_url":"https://patchwork.libcamera.org/comment/32506/","msgid":"","date":"2024-12-04T09:42:23","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > Hi Jacopo,\n> > >\n> > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > wrote:\n> > > >\n> > > > Hi Jacopo,\n> > > >\n> > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > wrote:\n> > > > >\n> > > > > Hi Harvey\n> > > > >\n> > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > From: Han-Lin Chen \n> > > > > >\n> > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > >\n> > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > Co-developed-by: Harvey Yang \n> > > > > > Signed-off-by: Harvey Yang \n> > > > > > ---\n> > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > src/android/camera_device.h | 2 ++\n> > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > >\n> > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > --- a/src/android/camera_device.cpp\n> > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > CameraStream *stream = iter->first;\n> > > > > > StreamBuffer *buffer = iter->second;\n> > > > > >\n> > > > > > + if (stream->isJpegStream()) {\n> > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > + }\n> > > > > > +\n> > > > >\n> > > > > no {} for single line statements\n> > >\n> > > Done\n> > >\n> > > > >\n> > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > if (!src) {\n> > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > }\n> > > > > >\n> > > > > > +/*\n> > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > + */\n> > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > + StreamBuffer *buffer) const\n> > > > > > +{\n> > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > +\n> > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > +\n> > > > > > + /*\n> > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > >\n> > > > > \\todo\n> > >\n> > > done\n> > >\n> > > > >\n> > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > >\n> > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > handling it and it sets it on the ISP.\n> > > > >\n> > > > > I wonder if you couldn't simply use AnalogueGain here\n> > >\n> > > I think the comment here is assuming the further changes that use\n> > > AnalogueGain directly here, while might not be needed in this patch...\n> > > Removed.\n> >\n> > I'm not sure I get this comment in full. I was suggesting to use\n> > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > keeping a \\todo\n>\n> Yeah I know, while I think using AnalogueGain or even a \\todo is\n> irrelevant in this patch. We may add a separate one to use\n> AnalogueGain directly :)\n>\n> >\n> > >\n> > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > following patches.\n> > >\n> > > > >\n> > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > + * the source of digital gains.\n> > > > > > + */\n> > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > +}\n> > > > > > +\n> > > > > > /*\n> > > > > > * Produce a set of fixed result metadata.\n> > > > > > */\n> > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > --- a/src/android/camera_device.h\n> > > > > > +++ b/src/android/camera_device.h\n> > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > StreamBuffer::Status status);\n> > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > + StreamBuffer *buffer) const;\n> > > > > > std::unique_ptr getResultMetadata(\n> > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > >\n> > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > --- a/src/android/camera_request.h\n> > > > > > +++ b/src/android/camera_request.h\n> > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > >\n> > > > > > + struct JpegExifMetadata {\n> > > > > > + int64_t sensorExposureTime;\n> > > > > > + int32_t sensorSensitivityISO;\n> > > > > > + };\n> > > > > > +\n> > > > > > CameraStream *stream;\n> > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > std::unique_ptr frameBuffer;\n> > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > Status status = Status::Success;\n> > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > std::unique_ptr dstBuffer;\n> > > > > > + std::optional jpegExifMetadata;\n> > > > > > Camera3RequestDescriptor *request;\n> > > > > >\n> > > > > > private:\n> > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > --- a/src/android/camera_stream.h\n> > > > > > +++ b/src/android/camera_stream.h\n> > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > libcamera::Stream *stream() const;\n> > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > + bool isJpegStream() const\n> > > > > > + {\n> > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > + }\n> > > > > >\n> > > > > > int configure();\n> > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > index f5a90785d..48782b574 100644\n> > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > >\n> > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > >\n> > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > >\n> > > > > This means it's not optional, isn't it ?\n> > >\n> > > Means it's not std::nullopt. Any suggestions?\n> >\n> > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > populated, so std::optional<> doesn't bring any value. But you\n> > probably want to make sure that jpegExifMetadata has been populated\n> > when this function is called, so feel free to keep std::optional<>\n>\n> Sure :)\n>\n> >\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > > >\n> > > > > >\n> > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > */\n> > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > >\n> > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > +\n> > > > >\n> > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > to. From there you could get the Request metadata as you currently do\n> > > > > in CameraDevice::generateJpegExifMetadata().\n> > > >\n> > > > That's a very good question, only that I wonder how we can handle the\n> > > > threading issue properly.\n> > > >\n> > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > to consider race conditions, and our goal is to support partial results,\n> > > > which means that the post processor thread might try to access metadata\n> > > > when the request is still being processed in the pipeline handler, which\n> > > > might set further metadata tags.\n> >\n> > Is this an issue about threading or about the order in which metadata\n> > and buffer completes ?\n>\n> What I meant is the threading issue, while the order is also an issue, true.\n>\n> >\n> > I see in the next patch you call generateJpegExifMetadata() on a\n> > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> >\n> > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> >\n> > And immediately after, if such a buffer has been completed, you call\n> > process() on it\n> > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> >\n> > Is this right ?\n> >\n> > So when the buffer completes, you inspect the so-far-completed\n> > metadata and extract ExposureTime (and AnalogueGain eventually).\n> >\n> > 1) Why you don't do that at metadataAvailable time ?\n>\n> Makes a lot of sense. Will try to apply in the next patch.\n>\n> > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > the time the buffer to process has completed ?\n>\n> That's a very good question. I'd like your input: Do you think we\n> should pause the jpeg process until all necessary metadata tags are\n> available?\n>\n\nGood question..\n\nAs I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\njpegExifMetadata metadata are used to populate the EXIF data. In the\nsame function the jpeg post-processor is run, so at the moment EXIF\nmetadata creation and post-processing happens at the same time and is in\nfacts run at requestComplete() time.\n\nLet me get the requirements straight here: I presume the idea is to\nproduce the jpeg frame as soon as:\n\n1) The source buffer is ready\n2) All the required metadata to populate the EXIF tags (collected by\nyour generateJpegExifMetadata()) are available\n\nwithout waiting for the whole request to complete ?\n\nIn your last patch you introduce support for handling\nCamera::bufferCompleted and the newly introduced\nCamera::metadataAvailable signals but the post-processing still\nhappens when the buffer is completed without verifying that the\nrequired metadata to populate the EXIF tags are available.\n\nDoes this match your understanding ?\n\nOne possible way to handle this is to\n1) at bufferCompleted time accumulate a list of completed FrameBuffers\nto process (and signal them with process_capture_request). Check if the\nRequest::metadata() where all the partial metadata are accumulated\ncontains the necessary tags for EXIF, if they do call process()\notherwise skip it\n2) at metadataAvailable time check if the EXIF are there, if they are\nwalk the list of completed buffers to process and run post-processing\n\nI'm sure there will be challanges, but what do you think of this as a\ngeneral outline ?\n\nNow, how to get there... I really think your last patch should be\nbroken out to pieces. It's too much stuff to digest in one go.\n\nThere are patches in this series that can be fast-tracked,\nspecifically the ones that make it possible to correctly handle\nmultiple Mapped stream on one Direct.\n\nI would take from this series:\n2/9\n3/9 + 4/9 squased together as suggested in the review\n5/9\n6/9\n7/9 (pending review, I asked Laurent to give there a look)\n\nThen, prepare a series to fast-track JPEG processing and support\npartial results. I would defer 1/9 to that series. Then introduce\nsupport for handling bufferCompleted() (without handling\nmetadataAvailable yet, this will validate that the HAL works with\nplatforms that do not deliver early metadata). Then handle\nmetadataAvailable on top.\n\nAgain, there will certainly be challanges, and I'm not even sure the\nbreakdown of the second part is possible, but indeed I would separate\n1/9 and 8/9 9/9 from the rest of the patches and fast-track the\nothers. What do you think ?\n\n\n> >\n> >\n> >\n> >\n> > > >\n> > > > WDYT?\n> > > >\n> > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > >\n> >\n> > I might have missed why this is related :)\n>\n> Okay, I think I misunderstood the \\todo, but the point is that it\n> doesn't handle the threading issue. We should avoid reading it in the\n> post processor thread, especially with partial results, as the Request\n> might not have been completed.\n>\n> BR,\n> Harvey\n>\n> >\n> > > > >\n> > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > CameraDevice::requestComplete() time ?\n> > > > >\n> > > > > Thanks\n> > > > > j\n> > > > >\n> > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > if (ret)\n> > > > > > exif.setAperture(*entry.data.f);\n> > > > > >\n> > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > -\n> > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > >\n> > > > > > --\n> > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 251B3BDB1C\n\tfor ;\n\tWed, 4 Dec 2024 09:42:29 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 450DC660A3;\n\tWed, 4 Dec 2024 10:42:28 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B970D618B5\n\tfor ;\n\tWed, 4 Dec 2024 10:42:26 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 869BB4D4;\n\tWed, 4 Dec 2024 10:41:58 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"XDpnACR/\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1733305318;\n\tbh=Ne7eLn5gyHo4T0izU2OwPLQvg0e4UrDNBDJUEsA7riM=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=XDpnACR/m51paiWLJHJ8FNTrIsQyW/qCm1EqrqAzpm8GQLcVt0nQP9zHSXQZf1QO/\n\tJEry5jx3eUwzBRIJ7wyw2q/A5NUJ6HBR/qEZOGvv7bm3MDE0j7lBHsuXs/fOxsiBbF\n\t2v6wklpSdXtCaB546HDkRMwm+zMOuLJlWnKSvo6A=","Date":"Wed, 4 Dec 2024 10:42:23 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32508,"web_url":"https://patchwork.libcamera.org/comment/32508/","msgid":"","date":"2024-12-04T10:29:53","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > wrote:\n> > > > >\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Harvey\n> > > > > >\n> > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > From: Han-Lin Chen \n> > > > > > >\n> > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > >\n> > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > ---\n> > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > >\n> > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > CameraStream *stream = iter->first;\n> > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > >\n> > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > + }\n> > > > > > > +\n> > > > > >\n> > > > > > no {} for single line statements\n> > > >\n> > > > Done\n> > > >\n> > > > > >\n> > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > if (!src) {\n> > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > }\n> > > > > > >\n> > > > > > > +/*\n> > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > + */\n> > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > + StreamBuffer *buffer) const\n> > > > > > > +{\n> > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > +\n> > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > +\n> > > > > > > + /*\n> > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > >\n> > > > > > \\todo\n> > > >\n> > > > done\n> > > >\n> > > > > >\n> > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > >\n> > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > handling it and it sets it on the ISP.\n> > > > > >\n> > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > >\n> > > > I think the comment here is assuming the further changes that use\n> > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > Removed.\n> > >\n> > > I'm not sure I get this comment in full. I was suggesting to use\n> > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > keeping a \\todo\n> >\n> > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > irrelevant in this patch. We may add a separate one to use\n> > AnalogueGain directly :)\n> >\n> > >\n> > > >\n> > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > following patches.\n> > > >\n> > > > > >\n> > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > + * the source of digital gains.\n> > > > > > > + */\n> > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > +}\n> > > > > > > +\n> > > > > > > /*\n> > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > */\n> > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > --- a/src/android/camera_device.h\n> > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > StreamBuffer::Status status);\n> > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > >\n> > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > --- a/src/android/camera_request.h\n> > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > >\n> > > > > > > + struct JpegExifMetadata {\n> > > > > > > + int64_t sensorExposureTime;\n> > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > + };\n> > > > > > > +\n> > > > > > > CameraStream *stream;\n> > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > Status status = Status::Success;\n> > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > Camera3RequestDescriptor *request;\n> > > > > > >\n> > > > > > > private:\n> > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > libcamera::Stream *stream() const;\n> > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > + bool isJpegStream() const\n> > > > > > > + {\n> > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > + }\n> > > > > > >\n> > > > > > > int configure();\n> > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > >\n> > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > >\n> > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > >\n> > > > > > This means it's not optional, isn't it ?\n> > > >\n> > > > Means it's not std::nullopt. Any suggestions?\n> > >\n> > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > populated, so std::optional<> doesn't bring any value. But you\n> > > probably want to make sure that jpegExifMetadata has been populated\n> > > when this function is called, so feel free to keep std::optional<>\n> >\n> > Sure :)\n> >\n> > >\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > > > >\n> > > > > > >\n> > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > */\n> > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > >\n> > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > +\n> > > > > >\n> > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > >\n> > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > threading issue properly.\n> > > > >\n> > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > which means that the post processor thread might try to access metadata\n> > > > > when the request is still being processed in the pipeline handler, which\n> > > > > might set further metadata tags.\n> > >\n> > > Is this an issue about threading or about the order in which metadata\n> > > and buffer completes ?\n> >\n> > What I meant is the threading issue, while the order is also an issue, true.\n> >\n> > >\n> > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > >\n> > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > >\n> > > And immediately after, if such a buffer has been completed, you call\n> > > process() on it\n> > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > >\n> > > Is this right ?\n> > >\n> > > So when the buffer completes, you inspect the so-far-completed\n> > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > >\n> > > 1) Why you don't do that at metadataAvailable time ?\n> >\n> > Makes a lot of sense. Will try to apply in the next patch.\n> >\n> > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > the time the buffer to process has completed ?\n> >\n> > That's a very good question. I'd like your input: Do you think we\n> > should pause the jpeg process until all necessary metadata tags are\n> > available?\n> >\n>\n> Good question..\n>\n> As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> jpegExifMetadata metadata are used to populate the EXIF data. In the\n> same function the jpeg post-processor is run, so at the moment EXIF\n> metadata creation and post-processing happens at the same time and is in\n> facts run at requestComplete() time.\n>\n> Let me get the requirements straight here: I presume the idea is to\n> produce the jpeg frame as soon as:\n>\n> 1) The source buffer is ready\n> 2) All the required metadata to populate the EXIF tags (collected by\n> your generateJpegExifMetadata()) are available\n>\n> without waiting for the whole request to complete ?\n>\n> In your last patch you introduce support for handling\n> Camera::bufferCompleted and the newly introduced\n> Camera::metadataAvailable signals but the post-processing still\n> happens when the buffer is completed without verifying that the\n> required metadata to populate the EXIF tags are available.\n>\n> Does this match your understanding ?\n\nYes, it's my understanding as well. Thanks for sorting the logic out.\n\nHan-lin proposes a question though: The ExposureTime (and the upcoming\nAnalogueGain) metadata should already be ready when the jpeg buffer is\ncompleted, so perhaps we could expect them to be notified with signal\nmetadataAvailable earlier than bufferCompleted being called? He\nsuggests adding a WARNING/ERROR log and a \\todo, although I feel that\nyou wouldn't like it(?\n\n\n\n>\n> One possible way to handle this is to\n> 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> to process (and signal them with process_capture_request). Check if the\n> Request::metadata() where all the partial metadata are accumulated\n> contains the necessary tags for EXIF, if they do call process()\n> otherwise skip it\n> 2) at metadataAvailable time check if the EXIF are there, if they are\n> walk the list of completed buffers to process and run post-processing\n\nYes, basically this is what I have in my mind as well, thanks.\n\n>\n> I'm sure there will be challanges, but what do you think of this as a\n> general outline ?\n>\n> Now, how to get there... I really think your last patch should be\n> broken out to pieces. It's too much stuff to digest in one go.\n>\n> There are patches in this series that can be fast-tracked,\n> specifically the ones that make it possible to correctly handle\n> multiple Mapped stream on one Direct.\n>\n> I would take from this series:\n> 2/9\n> 3/9 + 4/9 squased together as suggested in the review\n\nSure, I can squash them if you think that makes more sense. (I spent\nquite some time breaking this into two though in the beginning haha...\n\n> 5/9\n> 6/9\n\nI think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\nand the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n\n> 7/9 (pending review, I asked Laurent to give there a look)\n>\n> Then, prepare a series to fast-track JPEG processing and support\n> partial results. I would defer 1/9 to that series. Then introduce\n\nSure :)\n\n> support for handling bufferCompleted() (without handling\n> metadataAvailable yet, this will validate that the HAL works with\n> platforms that do not deliver early metadata). Then handle\n> metadataAvailable on top.\n\nI suggest to support signal metadataAvailable first, because:\n\n- It actually involves less changes, as a set of metadata wouldn't be\nblocked by post processing, and we could immediately notify the\napplication with a partial result.\n- metadataAvailable and bufferCompleted are actually both new to\nAndroid adapter.\n- I've actually done this split in this order, but haven't sent them\nas patches yet. You can check on gitlab first:\nhttps://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\nhttps://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n\n>\n> Again, there will certainly be challanges, and I'm not even sure the\n> breakdown of the second part is possible, but indeed I would separate\n> 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> others. What do you think ?\n\nSure, just one question: Do you think we should merge this Jpeg\nmetadata patch into the rest two of the upcoming patches\n(metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\nimplement it without Camera3ResultDescriptor and instances available.\n\nBR,\nHarvey\n\n>\n>\n> > >\n> > >\n> > >\n> > >\n> > > > >\n> > > > > WDYT?\n> > > > >\n> > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > >\n> > >\n> > > I might have missed why this is related :)\n> >\n> > Okay, I think I misunderstood the \\todo, but the point is that it\n> > doesn't handle the threading issue. We should avoid reading it in the\n> > post processor thread, especially with partial results, as the Request\n> > might not have been completed.\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > > > > >\n> > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > CameraDevice::requestComplete() time ?\n> > > > > >\n> > > > > > Thanks\n> > > > > > j\n> > > > > >\n> > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > if (ret)\n> > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > >\n> > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > -\n> > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > >\n> > > > > > > --\n> > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id C74ADBDB13\n\tfor ;\n\tWed, 4 Dec 2024 10:30:10 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id D582D660AD;\n\tWed, 4 Dec 2024 11:30:09 +0100 (CET)","from mail-lj1-x22a.google.com (mail-lj1-x22a.google.com\n\t[IPv6:2a00:1450:4864:20::22a])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 805656609E\n\tfor ;\n\tWed, 4 Dec 2024 11:30:07 +0100 (CET)","by mail-lj1-x22a.google.com with SMTP id\n\t38308e7fff4ca-2ffbfee94d7so54497451fa.3\n\tfor ;\n\tWed, 04 Dec 2024 02:30:07 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"kLxYO4ZF\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1733308207; x=1733913007;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=JIVs212JsuoDEctkpQm7VGSJfWroKzoMNsdmWU/Us1w=;\n\tb=kLxYO4ZFC8H9Fvs3vqKWzQlNDQGleEmn4KX5wENSlnH04fZZDjtO2BT0AVmNuX5Mgs\n\tjqGmpqaLKIsWCaw8gKef7jg5QS/oZVoPFcXCRhaso44XrgoGSUFQlbNAM4ISOKO5HagY\n\tCD0jgjQZMKa2ALXT/S0I7qc5/d4jOKjgwcjkc=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1733308207; x=1733913007;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=JIVs212JsuoDEctkpQm7VGSJfWroKzoMNsdmWU/Us1w=;\n\tb=k5CLrT4nLIv9QrqBPkcPvMSK5zYks8sO3ib8NcWyMS7LK897DnQ45Nlhu4oQoD0lPm\n\tx5CwiqoHs4gzwEvjKRL8kWLnDsJk+jfC6F1fm3yqlKaf13+idEVGRjhYuDNL3zCHVkh2\n\t59t1cklSnDVRjUrnRNOUQYwlUr4nHYjsMw3FcJrgP8bU5clNlZZsJNAqsdDXFtwepcYP\n\tqx6NJ9AjSAV0IIzzcfziHFwoo3B+W1nxoouU+8ZLJ7q9x/Jqo7KDWRYZUZJQnEMcX6O3\n\tRC6DsVpY70Ws8glwB4DuIB8luEQ6qB2GJujo087Nw9wDSjJUWHb3H2WH6SGkCVowdHuu\n\tVpQg==","X-Gm-Message-State":"AOJu0YwjXFo3UzThlOcn6RsxClIxjGNk3BvvAaq9atls7QUe1on9Dc+v\n\tMc04x0RQ9BpDdVAsTReWsZfaEn0RMCYGaLG2cE533+DRh/8/9wr6cSECr0WkNGLTgutn7lJHGgC\n\tC4Y0PPxQi9LYioUN9vz+hf93ZstnAB+uw1Ikn4CxNbv9sras7JF4X","X-Gm-Gg":"ASbGncuwvjz7yHhiz0YkmXpCT8CTP2pxDuOJGpOOdf58LxMfs3SzT+tTJ6AiCm0waRt\n\t9N8zkabJ9m3d64R0cnqcFM+2f2GXWbR81Xt/MAJ9F7a3zrswDQ9WfxYKPnw==","X-Google-Smtp-Source":"AGHT+IHJZYJ+ATkShHkOYuKthZdTp0FkW5/yKHhMEYKXhkwCWJ8oBSSz7WoY8weNMiyIwbSRxQpu6A3hyItZQDanK3Y=","X-Received":"by 2002:a05:651c:2120:b0:300:1448:c526 with SMTP id\n\t38308e7fff4ca-30014eac030mr10454641fa.37.1733308206393;\n\tWed, 04 Dec 2024 02:30:06 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Wed, 4 Dec 2024 18:29:53 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32509,"web_url":"https://patchwork.libcamera.org/comment/32509/","msgid":"","date":"2024-12-04T10:48:04","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > Hi Jacopo,\n> > >\n> > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Harvey\n> > > > > > >\n> > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > From: Han-Lin Chen \n> > > > > > > >\n> > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > >\n> > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > ---\n> > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > >\n> > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > >\n> > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > + }\n> > > > > > > > +\n> > > > > > >\n> > > > > > > no {} for single line statements\n> > > > >\n> > > > > Done\n> > > > >\n> > > > > > >\n> > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > if (!src) {\n> > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > }\n> > > > > > > >\n> > > > > > > > +/*\n> > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > + */\n> > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > +{\n> > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > +\n> > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > +\n> > > > > > > > + /*\n> > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > >\n> > > > > > > \\todo\n> > > > >\n> > > > > done\n> > > > >\n> > > > > > >\n> > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > >\n> > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > handling it and it sets it on the ISP.\n> > > > > > >\n> > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > >\n> > > > > I think the comment here is assuming the further changes that use\n> > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > Removed.\n> > > >\n> > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > keeping a \\todo\n> > >\n> > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > irrelevant in this patch. We may add a separate one to use\n> > > AnalogueGain directly :)\n> > >\n> > > >\n> > > > >\n> > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > following patches.\n> > > > >\n> > > > > > >\n> > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > + * the source of digital gains.\n> > > > > > > > + */\n> > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > +}\n> > > > > > > > +\n> > > > > > > > /*\n> > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > */\n> > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > StreamBuffer::Status status);\n> > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > >\n> > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > >\n> > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > + };\n> > > > > > > > +\n> > > > > > > > CameraStream *stream;\n> > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > Status status = Status::Success;\n> > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > >\n> > > > > > > > private:\n> > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > + bool isJpegStream() const\n> > > > > > > > + {\n> > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > + }\n> > > > > > > >\n> > > > > > > > int configure();\n> > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > >\n> > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > >\n> > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > >\n> > > > > > > This means it's not optional, isn't it ?\n> > > > >\n> > > > > Means it's not std::nullopt. Any suggestions?\n> > > >\n> > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > when this function is called, so feel free to keep std::optional<>\n> > >\n> > > Sure :)\n> > >\n> > > >\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > */\n> > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > >\n> > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > +\n> > > > > > >\n> > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > >\n> > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > threading issue properly.\n> > > > > >\n> > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > which means that the post processor thread might try to access metadata\n> > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > might set further metadata tags.\n> > > >\n> > > > Is this an issue about threading or about the order in which metadata\n> > > > and buffer completes ?\n> > >\n> > > What I meant is the threading issue, while the order is also an issue, true.\n> > >\n> > > >\n> > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > >\n> > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > >\n> > > > And immediately after, if such a buffer has been completed, you call\n> > > > process() on it\n> > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > >\n> > > > Is this right ?\n> > > >\n> > > > So when the buffer completes, you inspect the so-far-completed\n> > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > >\n> > > > 1) Why you don't do that at metadataAvailable time ?\n> > >\n> > > Makes a lot of sense. Will try to apply in the next patch.\n> > >\n> > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > the time the buffer to process has completed ?\n> > >\n> > > That's a very good question. I'd like your input: Do you think we\n> > > should pause the jpeg process until all necessary metadata tags are\n> > > available?\n> > >\n> >\n> > Good question..\n> >\n> > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > same function the jpeg post-processor is run, so at the moment EXIF\n> > metadata creation and post-processing happens at the same time and is in\n> > facts run at requestComplete() time.\n> >\n> > Let me get the requirements straight here: I presume the idea is to\n> > produce the jpeg frame as soon as:\n> >\n> > 1) The source buffer is ready\n> > 2) All the required metadata to populate the EXIF tags (collected by\n> > your generateJpegExifMetadata()) are available\n> >\n> > without waiting for the whole request to complete ?\n> >\n> > In your last patch you introduce support for handling\n> > Camera::bufferCompleted and the newly introduced\n> > Camera::metadataAvailable signals but the post-processing still\n> > happens when the buffer is completed without verifying that the\n> > required metadata to populate the EXIF tags are available.\n> >\n> > Does this match your understanding ?\n>\n> Yes, it's my understanding as well. Thanks for sorting the logic out.\n>\n> Han-lin proposes a question though: The ExposureTime (and the upcoming\n> AnalogueGain) metadata should already be ready when the jpeg buffer is\n> completed, so perhaps we could expect them to be notified with signal\n\nYes, \"should\", however there's nothing that forces a pipeline handler\nto guarantee that they are. In example, a pipeline handler (especially\nan existing one not ported to use the metadataAvailable signal) might not\nsupport early metadata completion but could signl buffer being\ncompleted. Unless we make it mandatory for pipelines to signal those\nmetadata before buffers and implement compliance tests to validate\nthat, I don't think we can assume anything about ordering.\n\n> metadataAvailable earlier than bufferCompleted being called? He\n> suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> you wouldn't like it(?\n>\n\nI don't think signalling metadata at requestComplete time -after-\nbufferCompleted is a warning/error condition. I'll check with other\nwhat they think about this.\n\n>\n>\n> >\n> > One possible way to handle this is to\n> > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > to process (and signal them with process_capture_request). Check if the\n> > Request::metadata() where all the partial metadata are accumulated\n> > contains the necessary tags for EXIF, if they do call process()\n> > otherwise skip it\n> > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > walk the list of completed buffers to process and run post-processing\n>\n> Yes, basically this is what I have in my mind as well, thanks.\n>\n> >\n> > I'm sure there will be challanges, but what do you think of this as a\n> > general outline ?\n> >\n> > Now, how to get there... I really think your last patch should be\n> > broken out to pieces. It's too much stuff to digest in one go.\n> >\n> > There are patches in this series that can be fast-tracked,\n> > specifically the ones that make it possible to correctly handle\n> > multiple Mapped stream on one Direct.\n> >\n> > I would take from this series:\n> > 2/9\n> > 3/9 + 4/9 squased together as suggested in the review\n>\n> Sure, I can squash them if you think that makes more sense. (I spent\n> quite some time breaking this into two though in the beginning haha...\n>\n\nSorry about that, but the patch is really about supporting multiple\nmapped streams, isn't it ?\n\n> > 5/9\n> > 6/9\n>\n> I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n>\n\nThanks\n\n> > 7/9 (pending review, I asked Laurent to give there a look)\n> >\n> > Then, prepare a series to fast-track JPEG processing and support\n> > partial results. I would defer 1/9 to that series. Then introduce\n>\n> Sure :)\n>\n> > support for handling bufferCompleted() (without handling\n> > metadataAvailable yet, this will validate that the HAL works with\n> > platforms that do not deliver early metadata). Then handle\n> > metadataAvailable on top.\n>\n> I suggest to support signal metadataAvailable first, because:\n>\n> - It actually involves less changes, as a set of metadata wouldn't be\n> blocked by post processing, and we could immediately notify the\n> application with a partial result.\n> - metadataAvailable and bufferCompleted are actually both new to\n> Android adapter.\n> - I've actually done this split in this order, but haven't sent them\n> as patches yet. You can check on gitlab first:\n> https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n>\n\nSure, as long as we test with CTS with both pipelines supporting\nmetadataAvailable and pipelines not supporting metadataAvailable\n\n> >\n> > Again, there will certainly be challanges, and I'm not even sure the\n> > breakdown of the second part is possible, but indeed I would separate\n> > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > others. What do you think ?\n>\n> Sure, just one question: Do you think we should merge this Jpeg\n> metadata patch into the rest two of the upcoming patches\n\nPlease note supporting metadataAvailable and bufferComplete in the HAL\nmight require more than just two patches :)\n\n> (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> implement it without Camera3ResultDescriptor and instances available.\n\nIdeally, after having plumbed in support for metadataAvailable and\nbufferComplete we should be in a state where \"processing Mapped\nstreams as soon as they're available\" should become quite natural to\nbe done on top.\n\nI presume you will need Camera3ResultDescriptor already when\nhandling (metadataAvailable & bufferCompleted) or am I wrong ?\n\n>\n> BR,\n> Harvey\n>\n> >\n> >\n> > > >\n> > > >\n> > > >\n> > > >\n> > > > > >\n> > > > > > WDYT?\n> > > > > >\n> > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > >\n> > > >\n> > > > I might have missed why this is related :)\n> > >\n> > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > doesn't handle the threading issue. We should avoid reading it in the\n> > > post processor thread, especially with partial results, as the Request\n> > > might not have been completed.\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > >\n> > > > > > >\n> > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > >\n> > > > > > > Thanks\n> > > > > > > j\n> > > > > > >\n> > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > if (ret)\n> > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > >\n> > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > -\n> > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > >\n> > > > > > > > --\n> > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id F1183BDB13\n\tfor ;\n\tWed, 4 Dec 2024 10:48:10 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id CE6C9660A3;\n\tWed, 4 Dec 2024 11:48:09 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DDBFE618B5\n\tfor ;\n\tWed, 4 Dec 2024 11:48:07 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id DB1766D6;\n\tWed, 4 Dec 2024 11:47:39 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"iFH34dFv\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1733309260;\n\tbh=AJlLjRi4fFlytpelSAJzsJTKE66PnG5r1kP3+/MK3F0=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=iFH34dFvFP9QF0ZOiwnzYYkJvxOoVm3LmmgIDo5j1UdY6ZkaIJ2VE3/rECXEokWSe\n\tR+Sheahg+J13hnoyFp1CwaHabBqXBLc34J3yuAObA25gvU0jAwCwEPzWjL/3yllMpp\n\tMbhAnHJOLRPdmuOYxOAnU4zaDRrFJOFQIhDzdqPI=","Date":"Wed, 4 Dec 2024 11:48:04 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32518,"web_url":"https://patchwork.libcamera.org/comment/32518/","msgid":"","date":"2024-12-04T14:31:16","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > wrote:\n> > > > >\n> > > > > Hi Harvey\n> > > > >\n> > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > wrote:\n> > > > > > > >\n> > > > > > > > Hi Harvey\n> > > > > > > >\n> > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > From: Han-Lin Chen \n> > > > > > > > >\n> > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > >\n> > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > ---\n> > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > >\n> > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > >\n> > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > + }\n> > > > > > > > > +\n> > > > > > > >\n> > > > > > > > no {} for single line statements\n> > > > > >\n> > > > > > Done\n> > > > > >\n> > > > > > > >\n> > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > if (!src) {\n> > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > +/*\n> > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > + */\n> > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > +{\n> > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > +\n> > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > +\n> > > > > > > > > + /*\n> > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > >\n> > > > > > > > \\todo\n> > > > > >\n> > > > > > done\n> > > > > >\n> > > > > > > >\n> > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > >\n> > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > >\n> > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > >\n> > > > > > I think the comment here is assuming the further changes that use\n> > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > Removed.\n> > > > >\n> > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > keeping a \\todo\n> > > >\n> > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > irrelevant in this patch. We may add a separate one to use\n> > > > AnalogueGain directly :)\n> > > >\n> > > > >\n> > > > > >\n> > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > following patches.\n> > > > > >\n> > > > > > > >\n> > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > + * the source of digital gains.\n> > > > > > > > > + */\n> > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > +}\n> > > > > > > > > +\n> > > > > > > > > /*\n> > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > */\n> > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > >\n> > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > >\n> > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > + };\n> > > > > > > > > +\n> > > > > > > > > CameraStream *stream;\n> > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > Status status = Status::Success;\n> > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > >\n> > > > > > > > > private:\n> > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > + {\n> > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > + }\n> > > > > > > > >\n> > > > > > > > > int configure();\n> > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > >\n> > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > >\n> > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > >\n> > > > > > > > This means it's not optional, isn't it ?\n> > > > > >\n> > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > >\n> > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > when this function is called, so feel free to keep std::optional<>\n> > > >\n> > > > Sure :)\n> > > >\n> > > > >\n> > > > > >\n> > > > > > BR,\n> > > > > > Harvey\n> > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > */\n> > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > >\n> > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > +\n> > > > > > > >\n> > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > >\n> > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > threading issue properly.\n> > > > > > >\n> > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > might set further metadata tags.\n> > > > >\n> > > > > Is this an issue about threading or about the order in which metadata\n> > > > > and buffer completes ?\n> > > >\n> > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > >\n> > > > >\n> > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > >\n> > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > >\n> > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > process() on it\n> > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > >\n> > > > > Is this right ?\n> > > > >\n> > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > >\n> > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > >\n> > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > >\n> > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > the time the buffer to process has completed ?\n> > > >\n> > > > That's a very good question. I'd like your input: Do you think we\n> > > > should pause the jpeg process until all necessary metadata tags are\n> > > > available?\n> > > >\n> > >\n> > > Good question..\n> > >\n> > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > metadata creation and post-processing happens at the same time and is in\n> > > facts run at requestComplete() time.\n> > >\n> > > Let me get the requirements straight here: I presume the idea is to\n> > > produce the jpeg frame as soon as:\n> > >\n> > > 1) The source buffer is ready\n> > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > your generateJpegExifMetadata()) are available\n> > >\n> > > without waiting for the whole request to complete ?\n> > >\n> > > In your last patch you introduce support for handling\n> > > Camera::bufferCompleted and the newly introduced\n> > > Camera::metadataAvailable signals but the post-processing still\n> > > happens when the buffer is completed without verifying that the\n> > > required metadata to populate the EXIF tags are available.\n> > >\n> > > Does this match your understanding ?\n> >\n> > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> >\n> > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > completed, so perhaps we could expect them to be notified with signal\n>\n> Yes, \"should\", however there's nothing that forces a pipeline handler\n> to guarantee that they are. In example, a pipeline handler (especially\n> an existing one not ported to use the metadataAvailable signal) might not\n> support early metadata completion but could signl buffer being\n> completed. Unless we make it mandatory for pipelines to signal those\n> metadata before buffers and implement compliance tests to validate\n> that, I don't think we can assume anything about ordering.\n>\n> > metadataAvailable earlier than bufferCompleted being called? He\n> > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > you wouldn't like it(?\n> >\n>\n> I don't think signalling metadata at requestComplete time -after-\n> bufferCompleted is a warning/error condition. I'll check with other\n> what they think about this.\n>\n> >\n> >\n> > >\n> > > One possible way to handle this is to\n> > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > to process (and signal them with process_capture_request). Check if the\n> > > Request::metadata() where all the partial metadata are accumulated\n> > > contains the necessary tags for EXIF, if they do call process()\n> > > otherwise skip it\n> > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > walk the list of completed buffers to process and run post-processing\n> >\n> > Yes, basically this is what I have in my mind as well, thanks.\n> >\n> > >\n> > > I'm sure there will be challanges, but what do you think of this as a\n> > > general outline ?\n> > >\n> > > Now, how to get there... I really think your last patch should be\n> > > broken out to pieces. It's too much stuff to digest in one go.\n> > >\n> > > There are patches in this series that can be fast-tracked,\n> > > specifically the ones that make it possible to correctly handle\n> > > multiple Mapped stream on one Direct.\n> > >\n> > > I would take from this series:\n> > > 2/9\n> > > 3/9 + 4/9 squased together as suggested in the review\n> >\n> > Sure, I can squash them if you think that makes more sense. (I spent\n> > quite some time breaking this into two though in the beginning haha...\n> >\n>\n> Sorry about that, but the patch is really about supporting multiple\n> mapped streams, isn't it ?\n\nYeah, and I still think that the 3rd patch doesn't contribute to\nsupporting multiple mapped streams...\nHaven't uploaded as a patch, but you can take a look at the new commit message:\nhttps://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n\nNo worries, I can still squash it.\n\n>\n> > > 5/9\n> > > 6/9\n> >\n> > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> >\n>\n> Thanks\n\nWe still have some ongoing discussion in the patch. Please help take a\nlook when you have time. Thanks!\n\n>\n> > > 7/9 (pending review, I asked Laurent to give there a look)\n> > >\n> > > Then, prepare a series to fast-track JPEG processing and support\n> > > partial results. I would defer 1/9 to that series. Then introduce\n> >\n> > Sure :)\n> >\n> > > support for handling bufferCompleted() (without handling\n> > > metadataAvailable yet, this will validate that the HAL works with\n> > > platforms that do not deliver early metadata). Then handle\n> > > metadataAvailable on top.\n> >\n> > I suggest to support signal metadataAvailable first, because:\n> >\n> > - It actually involves less changes, as a set of metadata wouldn't be\n> > blocked by post processing, and we could immediately notify the\n> > application with a partial result.\n> > - metadataAvailable and bufferCompleted are actually both new to\n> > Android adapter.\n> > - I've actually done this split in this order, but haven't sent them\n> > as patches yet. You can check on gitlab first:\n> > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> >\n>\n> Sure, as long as we test with CTS with both pipelines supporting\n> metadataAvailable and pipelines not supporting metadataAvailable\n\nBefore the whole series get merged, I'll test it on mtkisp7, which\nwill support the new signal. We also have ipu3 on soraka-libcamera\nthat doesn't support it yet.\n\n>\n> > >\n> > > Again, there will certainly be challanges, and I'm not even sure the\n> > > breakdown of the second part is possible, but indeed I would separate\n> > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > others. What do you think ?\n> >\n> > Sure, just one question: Do you think we should merge this Jpeg\n> > metadata patch into the rest two of the upcoming patches\n>\n> Please note supporting metadataAvailable and bufferComplete in the HAL\n> might require more than just two patches :)\n\nI really hope I can split them into pieces, while I'm running out of\nideas... If you can take a brief look and let me know how you expect\nthem to be split, I'd appreciate that a lot!\n\nSome random ideas:\nOn `android: Support partial results with metadataAvailable`:\n\n- Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\nthe only result.\n- Add `Camera3ResultDescriptor` and\n`Camera3RequestDescriptor::finalResult_` (with resultMetadata_\nmigrated), while I'm not sure if it makes sense...\n\n>\n> > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > implement it without Camera3ResultDescriptor and instances available.\n>\n> Ideally, after having plumbed in support for metadataAvailable and\n> bufferComplete we should be in a state where \"processing Mapped\n> streams as soon as they're available\" should become quite natural to\n> be done on top.\n>\n> I presume you will need Camera3ResultDescriptor already when\n> handling (metadataAvailable & bufferCompleted) or am I wrong ?\n\nYes, what I meant is that I find it difficult to keep this patch\nseparated from the upcoming patches, unless we implement it on top of\nthem. In this way though, getting the correct result metadata when\npost-processing with jpeg will be unsupported in the middle. Is that\nacceptable?\n\nBR,\nHarvey\n\n>\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > >\n> > > > >\n> > > > >\n> > > > >\n> > > > >\n> > > > > > >\n> > > > > > > WDYT?\n> > > > > > >\n> > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > >\n> > > > >\n> > > > > I might have missed why this is related :)\n> > > >\n> > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > post processor thread, especially with partial results, as the Request\n> > > > might not have been completed.\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > > >\n> > > > > > > >\n> > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > >\n> > > > > > > > Thanks\n> > > > > > > > j\n> > > > > > > >\n> > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > if (ret)\n> > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > >\n> > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > -\n> > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > >\n> > > > > > > > > --\n> > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 52ADCBDB1C\n\tfor ;\n\tWed, 4 Dec 2024 14:31:34 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 55CC1660C0;\n\tWed, 4 Dec 2024 15:31:32 +0100 (CET)","from mail-lj1-x233.google.com (mail-lj1-x233.google.com\n\t[IPv6:2a00:1450:4864:20::233])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 03030618B5\n\tfor ;\n\tWed, 4 Dec 2024 15:31:28 +0100 (CET)","by mail-lj1-x233.google.com with SMTP id\n\t38308e7fff4ca-2ffbf4580cbso72138291fa.2\n\tfor ;\n\tWed, 04 Dec 2024 06:31:28 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"VWJzZ8Bp\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1733322688; x=1733927488;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=yAK8kbrkxigI7Tp2Zody6LLwDwuDj6t9nof+gMtrARc=;\n\tb=VWJzZ8BpMM9noh/MOnSSTuh236GC0e7JdWXKQcUWFlvQQLCPgNe2vrYTurOJYm+Tc3\n\tEfe4Cm1EW6Mq7VvohSQNdyloUPTJ3/RbDy+OQK0xgtwJlQVHIoMnJw1ETtgzHmltWxDt\n\tkhiu1IOsuR3OeRZE7ByRXOHYOz6F6c7A0ATv0=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1733322688; x=1733927488;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=yAK8kbrkxigI7Tp2Zody6LLwDwuDj6t9nof+gMtrARc=;\n\tb=cofUrtPsy6fpHzPhiAvvS480Tu50V2yxBIGQOfJO6/v9d8X0DNqlkfBQGS8BpSxazX\n\tnLeEcovajTrE6rt/Yh1Wfdd6ZLGGT59YLaihUjjWecQlYzqpT3asxxNqYxDcmVQCLGow\n\touxzEd5hzYewO0BaRrGMn4HzY4AYqWwvblgJgCl77xGHQWuJVWRB6OCnLWkG4qo2HGBY\n\tnVJ4cUMO+WWBNw/zKHz7TdKhPWbvr+3dTaG3T2R5omdzVisGZ09dL931oGQ41kblrEkj\n\tskreSesMpWw5O6tCpPZvnJjujbCl+nuMzajmjAic4gSONiLN6FgrC9NZkG+txDorJw74\n\tKWHA==","X-Gm-Message-State":"AOJu0Yzme4DIqDmNcltYsxxPXCtTPHoWxa7SdVqJURUsKCNJ+eI9jHxD\n\t5tn3hh+f76en+OIH2RQ7L1WDEJM9vh8CHk/LhkZD5frNOIEjLEH8IJpVqALvxM0rkMYv5uaZ32I\n\tMl5pHYcTJAQMYTS5TxL5IEQmL4O2t1CVebAXX","X-Gm-Gg":"ASbGnctKiQsnukDTji3EAabZMH6ZjE1kY4AwVz+XSj7M7TWIaLJNsUIOHEFyF/WKjPh\n\t2ELnn356atg6Q1x60p1PrXLvcBk7Fu2CtyQTo4vYIw+ztw0eYr3d/1qGsyIotffaVXQ==","X-Google-Smtp-Source":"AGHT+IF3dmDYRhr3UAEF6cInjoYEC8biD44tmXbbdGDvFbSWP71ZDCXGk2hIJHIrcYNVq0T6Umm017TlF+X8vkHw1MY=","X-Received":"by 2002:a2e:b890:0:b0:2ff:9269:ac1a with SMTP id\n\t38308e7fff4ca-30009b8d8b9mr47713741fa.0.1733322687813;\n\tWed, 04 Dec 2024 06:31:27 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-1-chenghaoyang@chromium.org>\n\t<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Wed, 4 Dec 2024 22:31:16 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32520,"web_url":"https://patchwork.libcamera.org/comment/32520/","msgid":"<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>","date":"2024-12-04T15:15:59","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > Hi Jacopo,\n> > >\n> > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Harvey\n> > > > > >\n> > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > wrote:\n> > > > > > > >\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > wrote:\n> > > > > > > > >\n> > > > > > > > > Hi Harvey\n> > > > > > > > >\n> > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > >\n> > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > >\n> > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > ---\n> > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > >\n> > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > >\n> > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > + }\n> > > > > > > > > > +\n> > > > > > > > >\n> > > > > > > > > no {} for single line statements\n> > > > > > >\n> > > > > > > Done\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > if (!src) {\n> > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > }\n> > > > > > > > > >\n> > > > > > > > > > +/*\n> > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > + */\n> > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > +{\n> > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > +\n> > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > +\n> > > > > > > > > > + /*\n> > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > >\n> > > > > > > > > \\todo\n> > > > > > >\n> > > > > > > done\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > >\n> > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > >\n> > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > >\n> > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > Removed.\n> > > > > >\n> > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > keeping a \\todo\n> > > > >\n> > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > AnalogueGain directly :)\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > following patches.\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > + */\n> > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > +}\n> > > > > > > > > > +\n> > > > > > > > > > /*\n> > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > */\n> > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > >\n> > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > >\n> > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > + };\n> > > > > > > > > > +\n> > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > >\n> > > > > > > > > > private:\n> > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > + {\n> > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > + }\n> > > > > > > > > >\n> > > > > > > > > > int configure();\n> > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > >\n> > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > >\n> > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > >\n> > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > >\n> > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > >\n> > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > >\n> > > > > Sure :)\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > BR,\n> > > > > > > Harvey\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > */\n> > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > >\n> > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > +\n> > > > > > > > >\n> > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > >\n> > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > threading issue properly.\n> > > > > > > >\n> > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > might set further metadata tags.\n> > > > > >\n> > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > and buffer completes ?\n> > > > >\n> > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > >\n> > > > > >\n> > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > >\n> > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > >\n> > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > process() on it\n> > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > >\n> > > > > > Is this right ?\n> > > > > >\n> > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > >\n> > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > >\n> > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > >\n> > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > the time the buffer to process has completed ?\n> > > > >\n> > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > available?\n> > > > >\n> > > >\n> > > > Good question..\n> > > >\n> > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > metadata creation and post-processing happens at the same time and is in\n> > > > facts run at requestComplete() time.\n> > > >\n> > > > Let me get the requirements straight here: I presume the idea is to\n> > > > produce the jpeg frame as soon as:\n> > > >\n> > > > 1) The source buffer is ready\n> > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > your generateJpegExifMetadata()) are available\n> > > >\n> > > > without waiting for the whole request to complete ?\n> > > >\n> > > > In your last patch you introduce support for handling\n> > > > Camera::bufferCompleted and the newly introduced\n> > > > Camera::metadataAvailable signals but the post-processing still\n> > > > happens when the buffer is completed without verifying that the\n> > > > required metadata to populate the EXIF tags are available.\n> > > >\n> > > > Does this match your understanding ?\n> > >\n> > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > >\n> > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > completed, so perhaps we could expect them to be notified with signal\n> >\n> > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > to guarantee that they are. In example, a pipeline handler (especially\n> > an existing one not ported to use the metadataAvailable signal) might not\n> > support early metadata completion but could signl buffer being\n> > completed. Unless we make it mandatory for pipelines to signal those\n> > metadata before buffers and implement compliance tests to validate\n> > that, I don't think we can assume anything about ordering.\n> >\n> > > metadataAvailable earlier than bufferCompleted being called? He\n> > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > you wouldn't like it(?\n> > >\n> >\n> > I don't think signalling metadata at requestComplete time -after-\n> > bufferCompleted is a warning/error condition. I'll check with other\n> > what they think about this.\n> >\n> > >\n> > >\n> > > >\n> > > > One possible way to handle this is to\n> > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > to process (and signal them with process_capture_request). Check if the\n> > > > Request::metadata() where all the partial metadata are accumulated\n> > > > contains the necessary tags for EXIF, if they do call process()\n> > > > otherwise skip it\n> > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > walk the list of completed buffers to process and run post-processing\n> > >\n> > > Yes, basically this is what I have in my mind as well, thanks.\n> > >\n> > > >\n> > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > general outline ?\n> > > >\n> > > > Now, how to get there... I really think your last patch should be\n> > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > >\n> > > > There are patches in this series that can be fast-tracked,\n> > > > specifically the ones that make it possible to correctly handle\n> > > > multiple Mapped stream on one Direct.\n> > > >\n> > > > I would take from this series:\n> > > > 2/9\n> > > > 3/9 + 4/9 squased together as suggested in the review\n> > >\n> > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > quite some time breaking this into two though in the beginning haha...\n> > >\n> >\n> > Sorry about that, but the patch is really about supporting multiple\n> > mapped streams, isn't it ?\n>\n> Yeah, and I still think that the 3rd patch doesn't contribute to\n> supporting multiple mapped streams...\n\nDo you mean 3/9, right ?\n\nFeel free to keep it separate if you prefer, as long as 4/9 is clearly\nabout \"supporting multiple mapped streams\"\n\n> Haven't uploaded as a patch, but you can take a look at the new commit message:\n> https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n\nCareful this patch contains a bunch of unrelated refactories at the\nend\n\n>\n> No worries, I can still squash it.\n>\n> >\n> > > > 5/9\n> > > > 6/9\n> > >\n> > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > >\n> >\n> > Thanks\n>\n> We still have some ongoing discussion in the patch. Please help take a\n> look when you have time. Thanks!\n\nOn 5/9 and 6/9 ?\n\n5/9 I sent my R-b tag\n6/9 I suggested a new commit message but the rest is ok with me\n\n>\n> >\n> > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > >\n> > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > partial results. I would defer 1/9 to that series. Then introduce\n> > >\n> > > Sure :)\n> > >\n> > > > support for handling bufferCompleted() (without handling\n> > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > platforms that do not deliver early metadata). Then handle\n> > > > metadataAvailable on top.\n> > >\n> > > I suggest to support signal metadataAvailable first, because:\n> > >\n> > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > blocked by post processing, and we could immediately notify the\n> > > application with a partial result.\n\nack\n\n> > > - metadataAvailable and bufferCompleted are actually both new to\n> > > Android adapter.\n> > > - I've actually done this split in this order, but haven't sent them\n\nfine with this ordering\n\n> > > as patches yet. You can check on gitlab first:\n> > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > >\n> >\n> > Sure, as long as we test with CTS with both pipelines supporting\n> > metadataAvailable and pipelines not supporting metadataAvailable\n>\n> Before the whole series get merged, I'll test it on mtkisp7, which\n> will support the new signal. We also have ipu3 on soraka-libcamera\n> that doesn't support it yet.\n>\n\nThat's great, thanks\n\n> >\n> > > >\n> > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > breakdown of the second part is possible, but indeed I would separate\n> > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > others. What do you think ?\n> > >\n> > > Sure, just one question: Do you think we should merge this Jpeg\n> > > metadata patch into the rest two of the upcoming patches\n> >\n> > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > might require more than just two patches :)\n>\n> I really hope I can split them into pieces, while I'm running out of\n> ideas... If you can take a brief look and let me know how you expect\n> them to be split, I'd appreciate that a lot!\n>\n> Some random ideas:\n> On `android: Support partial results with metadataAvailable`:\n>\n> - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> the only result.\n> - Add `Camera3ResultDescriptor` and\n> `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> migrated), while I'm not sure if it makes sense...\n>\n\nsee below\n\n> >\n> > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > implement it without Camera3ResultDescriptor and instances available.\n> >\n> > Ideally, after having plumbed in support for metadataAvailable and\n> > bufferComplete we should be in a state where \"processing Mapped\n> > streams as soon as they're available\" should become quite natural to\n> > be done on top.\n> >\n> > I presume you will need Camera3ResultDescriptor already when\n> > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n>\n> Yes, what I meant is that I find it difficult to keep this patch\n> separated from the upcoming patches, unless we implement it on top of\n> them. In this way though, getting the correct result metadata when\n\nIndeed I think it should happen after support for partial metadata and\nbufferCompleted have been developed\n\n> post-processing with jpeg will be unsupported in the middle. Is that\n\nWhy do you think so ? As long as process() happens at requestCompleted\ntime, all the metadata info you need will be available in\nRequest::metadata\n\n> acceptable?\n\nI still fail to see why doing this in lock-step would bring issues,\nbut I admit I only glanced through\n 8 files changed, 621 insertions(+), 277 deletions(-)\n\nCould you please tell me in which of the following steps you'll have\nissues:\n\n1) Add support for metadataAvailable signal\n\n Support the new signal and allow partial metadata completion. The\n HAL can now call process_capture_results multiple times for the\n same request.\n\n Post-processing is not changed as it happens at requestCompleted\n time, when the buffers are available and the required metadata to\n populate EXIF will be available in Request::metadata() as it used\n to be\n\n2) Add support for bufferCompleted\n\n Support earlier buffer completion. The HAL should already be\n instrumented to send partial results for the Direct buffer that has\n just completed.\n\n3) Schedule post-processing as soon as the source buffer is available\n\n Track if for each request the required metadata to populate the EXIF\n tags are available (for JPEG) and schedule post-processing as soon\n as the source buffer (and metadata for JPEG) are available.\n\n This indeed might require multiple patches to implement tracking of\n the metadata and buffer status\n\nAll steps should be validated with a pipeline that support\nmetadataAvailable and with a pipeline that doesn't.\n\nAs far as I can all of this currently happens in a single patch (9/9\nand partially in 8/9). I might be surely missing details and\nimplementation issues, but logically the above seems to me a\nreasonable break-down of 8/9 and 9/9 ?\n\n>\n> BR,\n> Harvey\n>\n> >\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > >\n> > > >\n> > > > > >\n> > > > > >\n> > > > > >\n> > > > > >\n> > > > > > > >\n> > > > > > > > WDYT?\n> > > > > > > >\n> > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > >\n> > > > > >\n> > > > > > I might have missed why this is related :)\n> > > > >\n> > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > post processor thread, especially with partial results, as the Request\n> > > > > might not have been completed.\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > > >\n> > > > > > > > >\n> > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > >\n> > > > > > > > > Thanks\n> > > > > > > > > j\n> > > > > > > > >\n> > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > if (ret)\n> > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > >\n> > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > -\n> > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > >\n> > > > > > > > > > --\n> > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id CD2FCBDB13\n\tfor ;\n\tWed, 4 Dec 2024 15:16:07 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C7168660C4;\n\tWed, 4 Dec 2024 16:16:05 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 5DD82618B5\n\tfor ;\n\tWed, 4 Dec 2024 16:16:03 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id DD6A64D4;\n\tWed, 4 Dec 2024 16:15:33 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"daRX391k\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1733325334;\n\tbh=c+N9f0WOh6l4tLHmA2jS1mYMSp2IF0IbwztP7hqjjtw=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=daRX391kIg53C15I5RXURPEmcz1/pXES4L+UB4rTP+BRHwxAZdi2BETrhY3uOOTkT\n\tpeyme+KffqBx0B/qZZ8l/4iKu39g2O+LjVzdarb+3ChKczPn0dYGaCN946QaNa7oEC\n\toqoXD/L1qVyYm/8pvfwi72GpY41NYpuDRZF/7j1E=","Date":"Wed, 4 Dec 2024 16:15:59 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>","References":"<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32521,"web_url":"https://patchwork.libcamera.org/comment/32521/","msgid":"","date":"2024-12-04T15:35:53","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > wrote:\n> > > > >\n> > > > > Hi Harvey\n> > > > >\n> > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Harvey\n> > > > > > >\n> > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > wrote:\n> > > > > > > > >\n> > > > > > > > > Hi Jacopo,\n> > > > > > > > >\n> > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > wrote:\n> > > > > > > > > >\n> > > > > > > > > > Hi Harvey\n> > > > > > > > > >\n> > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > >\n> > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > >\n> > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > ---\n> > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > >\n> > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > >\n> > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > + }\n> > > > > > > > > > > +\n> > > > > > > > > >\n> > > > > > > > > > no {} for single line statements\n> > > > > > > >\n> > > > > > > > Done\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > if (!src) {\n> > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > }\n> > > > > > > > > > >\n> > > > > > > > > > > +/*\n> > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > + */\n> > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > +{\n> > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > +\n> > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > +\n> > > > > > > > > > > + /*\n> > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > >\n> > > > > > > > > > \\todo\n> > > > > > > >\n> > > > > > > > done\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > >\n> > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > >\n> > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > >\n> > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > Removed.\n> > > > > > >\n> > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > keeping a \\todo\n> > > > > >\n> > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > AnalogueGain directly :)\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > following patches.\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > + */\n> > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > +}\n> > > > > > > > > > > +\n> > > > > > > > > > > /*\n> > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > */\n> > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > >\n> > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > >\n> > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > + };\n> > > > > > > > > > > +\n> > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > >\n> > > > > > > > > > > private:\n> > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > + {\n> > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > + }\n> > > > > > > > > > >\n> > > > > > > > > > > int configure();\n> > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > >\n> > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > >\n> > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > >\n> > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > >\n> > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > >\n> > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > >\n> > > > > > Sure :)\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > BR,\n> > > > > > > > Harvey\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > */\n> > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > >\n> > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > +\n> > > > > > > > > >\n> > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > >\n> > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > threading issue properly.\n> > > > > > > > >\n> > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > might set further metadata tags.\n> > > > > > >\n> > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > and buffer completes ?\n> > > > > >\n> > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > >\n> > > > > > >\n> > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > >\n> > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > >\n> > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > process() on it\n> > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > >\n> > > > > > > Is this right ?\n> > > > > > >\n> > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > >\n> > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > >\n> > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > >\n> > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > the time the buffer to process has completed ?\n> > > > > >\n> > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > available?\n> > > > > >\n> > > > >\n> > > > > Good question..\n> > > > >\n> > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > facts run at requestComplete() time.\n> > > > >\n> > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > produce the jpeg frame as soon as:\n> > > > >\n> > > > > 1) The source buffer is ready\n> > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > your generateJpegExifMetadata()) are available\n> > > > >\n> > > > > without waiting for the whole request to complete ?\n> > > > >\n> > > > > In your last patch you introduce support for handling\n> > > > > Camera::bufferCompleted and the newly introduced\n> > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > happens when the buffer is completed without verifying that the\n> > > > > required metadata to populate the EXIF tags are available.\n> > > > >\n> > > > > Does this match your understanding ?\n> > > >\n> > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > >\n> > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > completed, so perhaps we could expect them to be notified with signal\n> > >\n> > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > to guarantee that they are. In example, a pipeline handler (especially\n> > > an existing one not ported to use the metadataAvailable signal) might not\n> > > support early metadata completion but could signl buffer being\n> > > completed. Unless we make it mandatory for pipelines to signal those\n> > > metadata before buffers and implement compliance tests to validate\n> > > that, I don't think we can assume anything about ordering.\n> > >\n> > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > you wouldn't like it(?\n> > > >\n> > >\n> > > I don't think signalling metadata at requestComplete time -after-\n> > > bufferCompleted is a warning/error condition. I'll check with other\n> > > what they think about this.\n> > >\n> > > >\n> > > >\n> > > > >\n> > > > > One possible way to handle this is to\n> > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > otherwise skip it\n> > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > walk the list of completed buffers to process and run post-processing\n> > > >\n> > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > >\n> > > > >\n> > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > general outline ?\n> > > > >\n> > > > > Now, how to get there... I really think your last patch should be\n> > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > >\n> > > > > There are patches in this series that can be fast-tracked,\n> > > > > specifically the ones that make it possible to correctly handle\n> > > > > multiple Mapped stream on one Direct.\n> > > > >\n> > > > > I would take from this series:\n> > > > > 2/9\n> > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > >\n> > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > quite some time breaking this into two though in the beginning haha...\n> > > >\n> > >\n> > > Sorry about that, but the patch is really about supporting multiple\n> > > mapped streams, isn't it ?\n> >\n> > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > supporting multiple mapped streams...\n>\n> Do you mean 3/9, right ?\n>\n> Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> about \"supporting multiple mapped streams\"\n\nAh okay, thanks :)\n\n>\n> > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n>\n> Careful this patch contains a bunch of unrelated refactories at the\n> end\n\nYeah, my linter on nvim insists on cleaning up the whole file. I'll\nremove it when uploading :)\n\n>\n> >\n> > No worries, I can still squash it.\n> >\n> > >\n> > > > > 5/9\n> > > > > 6/9\n> > > >\n> > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > >\n> > >\n> > > Thanks\n> >\n> > We still have some ongoing discussion in the patch. Please help take a\n> > look when you have time. Thanks!\n>\n> On 5/9 and 6/9 ?\n>\n> 5/9 I sent my R-b tag\n> 6/9 I suggested a new commit message but the rest is ok with me\n\nYeah on 6/9 I assumed you would have questions about introducing\nCAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n\n>\n> >\n> > >\n> > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > >\n> > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > >\n> > > > Sure :)\n> > > >\n> > > > > support for handling bufferCompleted() (without handling\n> > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > platforms that do not deliver early metadata). Then handle\n> > > > > metadataAvailable on top.\n> > > >\n> > > > I suggest to support signal metadataAvailable first, because:\n> > > >\n> > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > blocked by post processing, and we could immediately notify the\n> > > > application with a partial result.\n>\n> ack\n>\n> > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > Android adapter.\n> > > > - I've actually done this split in this order, but haven't sent them\n>\n> fine with this ordering\n>\n> > > > as patches yet. You can check on gitlab first:\n> > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > >\n> > >\n> > > Sure, as long as we test with CTS with both pipelines supporting\n> > > metadataAvailable and pipelines not supporting metadataAvailable\n> >\n> > Before the whole series get merged, I'll test it on mtkisp7, which\n> > will support the new signal. We also have ipu3 on soraka-libcamera\n> > that doesn't support it yet.\n> >\n>\n> That's great, thanks\n>\n> > >\n> > > > >\n> > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > others. What do you think ?\n> > > >\n> > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > metadata patch into the rest two of the upcoming patches\n> > >\n> > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > might require more than just two patches :)\n> >\n> > I really hope I can split them into pieces, while I'm running out of\n> > ideas... If you can take a brief look and let me know how you expect\n> > them to be split, I'd appreciate that a lot!\n> >\n> > Some random ideas:\n> > On `android: Support partial results with metadataAvailable`:\n> >\n> > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > the only result.\n> > - Add `Camera3ResultDescriptor` and\n> > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > migrated), while I'm not sure if it makes sense...\n> >\n>\n> see below\n>\n> > >\n> > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > implement it without Camera3ResultDescriptor and instances available.\n> > >\n> > > Ideally, after having plumbed in support for metadataAvailable and\n> > > bufferComplete we should be in a state where \"processing Mapped\n> > > streams as soon as they're available\" should become quite natural to\n> > > be done on top.\n> > >\n> > > I presume you will need Camera3ResultDescriptor already when\n> > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> >\n> > Yes, what I meant is that I find it difficult to keep this patch\n> > separated from the upcoming patches, unless we implement it on top of\n> > them. In this way though, getting the correct result metadata when\n>\n> Indeed I think it should happen after support for partial metadata and\n> bufferCompleted have been developed\n>\n> > post-processing with jpeg will be unsupported in the middle. Is that\n>\n> Why do you think so ? As long as process() happens at requestCompleted\n> time, all the metadata info you need will be available in\n> Request::metadata\n>\n> > acceptable?\n>\n> I still fail to see why doing this in lock-step would bring issues,\n> but I admit I only glanced through\n> 8 files changed, 621 insertions(+), 277 deletions(-)\n>\n> Could you please tell me in which of the following steps you'll have\n> issues:\n>\n> 1) Add support for metadataAvailable signal\n>\n> Support the new signal and allow partial metadata completion. The\n> HAL can now call process_capture_results multiple times for the\n> same request.\n>\n> Post-processing is not changed as it happens at requestCompleted\n> time, when the buffers are available and the required metadata to\n> populate EXIF will be available in Request::metadata() as it used\n> to be\n>\n> 2) Add support for bufferCompleted\n>\n> Support earlier buffer completion. The HAL should already be\n> instrumented to send partial results for the Direct buffer that has\n> just completed.\n\nYes, if we keep the processing (or at least the jpeg one) in\nrequestComplete, then we won't drop the feature.\n\n>\n> 3) Schedule post-processing as soon as the source buffer is available\n>\n> Track if for each request the required metadata to populate the EXIF\n> tags are available (for JPEG) and schedule post-processing as soon\n> as the source buffer (and metadata for JPEG) are available.\n>\n> This indeed might require multiple patches to implement tracking of\n> the metadata and buffer status\n>\n> All steps should be validated with a pipeline that support\n> metadataAvailable and with a pipeline that doesn't.\n>\n> As far as I can all of this currently happens in a single patch (9/9\n> and partially in 8/9). I might be surely missing details and\n> implementation issues, but logically the above seems to me a\n> reasonable break-down of 8/9 and 9/9 ?\n\nYes, I'll try to implement this. Thanks!\n\nBR,\nHarvey\n\n>\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > > >\n> > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > WDYT?\n> > > > > > > > >\n> > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > >\n> > > > > > >\n> > > > > > > I might have missed why this is related :)\n> > > > > >\n> > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > might not have been completed.\n> > > > > >\n> > > > > > BR,\n> > > > > > Harvey\n> > > > > >\n> > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > >\n> > > > > > > > > > Thanks\n> > > > > > > > > > j\n> > > > > > > > > >\n> > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > if (ret)\n> > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > >\n> > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > -\n> > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > >\n> > > > > > > > > > > --\n> > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id DEABDBDB1C\n\tfor ;\n\tWed, 4 Dec 2024 15:36:10 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 2BDC3618B5;\n\tWed, 4 Dec 2024 16:36:09 +0100 (CET)","from mail-lj1-x22a.google.com (mail-lj1-x22a.google.com\n\t[IPv6:2a00:1450:4864:20::22a])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 45437618B5\n\tfor ;\n\tWed, 4 Dec 2024 16:36:07 +0100 (CET)","by mail-lj1-x22a.google.com with SMTP id\n\t38308e7fff4ca-2ffc86948dcso71316761fa.2\n\tfor ;\n\tWed, 04 Dec 2024 07:36:07 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"m7TZ92/S\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1733326565; x=1733931365;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=vNikI2sZykippKlohFXoULjQr8hpCa4zh4ohb8SbA10=;\n\tb=m7TZ92/SY45xRb2JLG6OAB8wfg8sz1kJSMLlQy2T9wQgm65TbS05upcP00dFIeo7Kh\n\tUAWj1oax5FmvAiBUzErY82EuN0ikfXwD/rMfvgkCmk+BMvtybzAXkZewda5IIWWm2Ut0\n\tkVVtcynan/inhuox5kdytCMRkt90iI1+d6RZQ=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1733326565; x=1733931365;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=vNikI2sZykippKlohFXoULjQr8hpCa4zh4ohb8SbA10=;\n\tb=ML2DoG0ErPxnqJmoAKGdaGjvnel5KBfORlN2PbPnX+AOnN63/3Pfb4shWgRy6MiRrx\n\trUOnsTQmMrEvnd1bJSaPRn4u/4MvB+wcf/s+Y5KnviPfogDeJ7F15Gv4LSPxwSpcjOtX\n\t3DSbEUMyDY1lw0n6Ka1oYonMF94SyGmM46HaADkn/YI//Un7F7LTC8dekuq0JgjZW7wp\n\tHRt+Btrd7gjA0Dzmv+D9K/Qka731w0xrmUADOloQ9vNw0HNZpK0Uj+VTwLjOk9GwIEhU\n\tq6+/XQP2/LzzAwmA0hdQzWgfcz5No/uCXSEkyER7liOtX2Z0i1h/s1EM6TySXaDGCElm\n\tdGFA==","X-Gm-Message-State":"AOJu0YyxdO/MYsOpRMdhrE3BtgLearvff5l5XxLzepF/AhmPk2KhOOef\n\tOS5ERogmhJNe1XS7N5gByPH62ggRT8GdxZGkPub9AMdAOmAI1sX7V1+Pc3Vfva5WCwzp+9qw3rk\n\tS3WcEMs5Hx5fWyKhOjki7NBlU6qXrxBLq8SYH","X-Gm-Gg":"ASbGncsP+e5nBcHwhHlwYkzlaY4P7voxq4LRQ3DJ9j8rHTwwUDZXSOcIeTgeJIMGBUq\n\tdkdwpLiuQbDyjcxznuhBVdrLJf2qI6hoAs+jtmT8ijtL/9ORLYRbTOMotWi5VM83tjg==","X-Google-Smtp-Source":"AGHT+IFig19ZWM2+6g4ZekH7/BLnjb9SH9nJODfKVtHmkxXECi4x0R/4TymCb4bPVcw8E7Xwn9UshBLK8v/S8yviQh0=","X-Received":"by 2002:a2e:bd13:0:b0:2ff:c3f6:a30c with SMTP id\n\t38308e7fff4ca-30009bf6233mr49002101fa.3.1733326564984;\n\tWed, 04 Dec 2024 07:36:04 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>","In-Reply-To":"<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>","From":"Cheng-Hao Yang ","Date":"Wed, 4 Dec 2024 23:35:53 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32686,"web_url":"https://patchwork.libcamera.org/comment/32686/","msgid":"","date":"2024-12-12T09:00:23","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n wrote:\n>\n> Hi Jacopo,\n>\n> On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > Hi Jacopo,\n> > >\n> > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Harvey\n> > > > > >\n> > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > wrote:\n> > > > > > > >\n> > > > > > > > Hi Harvey\n> > > > > > > >\n> > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > Hi Jacopo,\n> > > > > > > > >\n> > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > wrote:\n> > > > > > > > > >\n> > > > > > > > > > Hi Jacopo,\n> > > > > > > > > >\n> > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > wrote:\n> > > > > > > > > > >\n> > > > > > > > > > > Hi Harvey\n> > > > > > > > > > >\n> > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > >\n> > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > >\n> > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > ---\n> > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > >\n> > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > >\n> > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > + }\n> > > > > > > > > > > > +\n> > > > > > > > > > >\n> > > > > > > > > > > no {} for single line statements\n> > > > > > > > >\n> > > > > > > > > Done\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > }\n> > > > > > > > > > > >\n> > > > > > > > > > > > +/*\n> > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > + */\n> > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > +{\n> > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > +\n> > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > +\n> > > > > > > > > > > > + /*\n> > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > >\n> > > > > > > > > > > \\todo\n> > > > > > > > >\n> > > > > > > > > done\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > >\n> > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > >\n> > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > >\n> > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > Removed.\n> > > > > > > >\n> > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > keeping a \\todo\n> > > > > > >\n> > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > AnalogueGain directly :)\n> > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > following patches.\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > + */\n> > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > +}\n> > > > > > > > > > > > +\n> > > > > > > > > > > > /*\n> > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > */\n> > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > >\n> > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > >\n> > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > + };\n> > > > > > > > > > > > +\n> > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > >\n> > > > > > > > > > > > private:\n> > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > + {\n> > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > + }\n> > > > > > > > > > > >\n> > > > > > > > > > > > int configure();\n> > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > >\n> > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > >\n> > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > >\n> > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > >\n> > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > >\n> > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > >\n> > > > > > > Sure :)\n> > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > BR,\n> > > > > > > > > Harvey\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > */\n> > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > >\n> > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > +\n> > > > > > > > > > >\n> > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > >\n> > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > threading issue properly.\n> > > > > > > > > >\n> > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > might set further metadata tags.\n> > > > > > > >\n> > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > and buffer completes ?\n> > > > > > >\n> > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > >\n> > > > > > > >\n> > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > >\n> > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > >\n> > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > process() on it\n> > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > >\n> > > > > > > > Is this right ?\n> > > > > > > >\n> > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > >\n> > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > >\n> > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > >\n> > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > the time the buffer to process has completed ?\n> > > > > > >\n> > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > available?\n> > > > > > >\n> > > > > >\n> > > > > > Good question..\n> > > > > >\n> > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > facts run at requestComplete() time.\n> > > > > >\n> > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > produce the jpeg frame as soon as:\n> > > > > >\n> > > > > > 1) The source buffer is ready\n> > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > your generateJpegExifMetadata()) are available\n> > > > > >\n> > > > > > without waiting for the whole request to complete ?\n> > > > > >\n> > > > > > In your last patch you introduce support for handling\n> > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > happens when the buffer is completed without verifying that the\n> > > > > > required metadata to populate the EXIF tags are available.\n> > > > > >\n> > > > > > Does this match your understanding ?\n> > > > >\n> > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > >\n> > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > completed, so perhaps we could expect them to be notified with signal\n> > > >\n> > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > support early metadata completion but could signl buffer being\n> > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > metadata before buffers and implement compliance tests to validate\n> > > > that, I don't think we can assume anything about ordering.\n> > > >\n> > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > you wouldn't like it(?\n> > > > >\n> > > >\n> > > > I don't think signalling metadata at requestComplete time -after-\n> > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > what they think about this.\n> > > >\n> > > > >\n> > > > >\n> > > > > >\n> > > > > > One possible way to handle this is to\n> > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > otherwise skip it\n> > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > walk the list of completed buffers to process and run post-processing\n> > > > >\n> > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > >\n> > > > > >\n> > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > general outline ?\n> > > > > >\n> > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > >\n> > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > multiple Mapped stream on one Direct.\n> > > > > >\n> > > > > > I would take from this series:\n> > > > > > 2/9\n> > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > >\n> > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > quite some time breaking this into two though in the beginning haha...\n> > > > >\n> > > >\n> > > > Sorry about that, but the patch is really about supporting multiple\n> > > > mapped streams, isn't it ?\n> > >\n> > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > supporting multiple mapped streams...\n> >\n> > Do you mean 3/9, right ?\n> >\n> > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > about \"supporting multiple mapped streams\"\n>\n> Ah okay, thanks :)\n>\n> >\n> > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> >\n> > Careful this patch contains a bunch of unrelated refactories at the\n> > end\n>\n> Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> remove it when uploading :)\n>\n> >\n> > >\n> > > No worries, I can still squash it.\n> > >\n> > > >\n> > > > > > 5/9\n> > > > > > 6/9\n> > > > >\n> > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > >\n> > > >\n> > > > Thanks\n> > >\n> > > We still have some ongoing discussion in the patch. Please help take a\n> > > look when you have time. Thanks!\n> >\n> > On 5/9 and 6/9 ?\n> >\n> > 5/9 I sent my R-b tag\n> > 6/9 I suggested a new commit message but the rest is ok with me\n>\n> Yeah on 6/9 I assumed you would have questions about introducing\n> CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n>\n> >\n> > >\n> > > >\n> > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > >\n> > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > >\n> > > > > Sure :)\n> > > > >\n> > > > > > support for handling bufferCompleted() (without handling\n> > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > metadataAvailable on top.\n> > > > >\n> > > > > I suggest to support signal metadataAvailable first, because:\n> > > > >\n> > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > blocked by post processing, and we could immediately notify the\n> > > > > application with a partial result.\n> >\n> > ack\n> >\n> > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > Android adapter.\n> > > > > - I've actually done this split in this order, but haven't sent them\n> >\n> > fine with this ordering\n> >\n> > > > > as patches yet. You can check on gitlab first:\n> > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > >\n> > > >\n> > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > >\n> > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > that doesn't support it yet.\n> > >\n> >\n> > That's great, thanks\n> >\n> > > >\n> > > > > >\n> > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > others. What do you think ?\n> > > > >\n> > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > metadata patch into the rest two of the upcoming patches\n> > > >\n> > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > might require more than just two patches :)\n> > >\n> > > I really hope I can split them into pieces, while I'm running out of\n> > > ideas... If you can take a brief look and let me know how you expect\n> > > them to be split, I'd appreciate that a lot!\n> > >\n> > > Some random ideas:\n> > > On `android: Support partial results with metadataAvailable`:\n> > >\n> > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > the only result.\n> > > - Add `Camera3ResultDescriptor` and\n> > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > migrated), while I'm not sure if it makes sense...\n> > >\n> >\n> > see below\n> >\n> > > >\n> > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > >\n> > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > streams as soon as they're available\" should become quite natural to\n> > > > be done on top.\n> > > >\n> > > > I presume you will need Camera3ResultDescriptor already when\n> > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > >\n> > > Yes, what I meant is that I find it difficult to keep this patch\n> > > separated from the upcoming patches, unless we implement it on top of\n> > > them. In this way though, getting the correct result metadata when\n> >\n> > Indeed I think it should happen after support for partial metadata and\n> > bufferCompleted have been developed\n> >\n> > > post-processing with jpeg will be unsupported in the middle. Is that\n> >\n> > Why do you think so ? As long as process() happens at requestCompleted\n> > time, all the metadata info you need will be available in\n> > Request::metadata\n> >\n> > > acceptable?\n> >\n> > I still fail to see why doing this in lock-step would bring issues,\n> > but I admit I only glanced through\n> > 8 files changed, 621 insertions(+), 277 deletions(-)\n> >\n> > Could you please tell me in which of the following steps you'll have\n> > issues:\n> >\n> > 1) Add support for metadataAvailable signal\n> >\n> > Support the new signal and allow partial metadata completion. The\n> > HAL can now call process_capture_results multiple times for the\n> > same request.\n> >\n> > Post-processing is not changed as it happens at requestCompleted\n> > time, when the buffers are available and the required metadata to\n> > populate EXIF will be available in Request::metadata() as it used\n> > to be\n> >\n> > 2) Add support for bufferCompleted\n> >\n> > Support earlier buffer completion. The HAL should already be\n> > instrumented to send partial results for the Direct buffer that has\n> > just completed.\n\nI assume this includes sending the direct buffers back to the\napplication when receiving signals of bufferCompleted, while delaying\npost-processing to requestCompleted.\n\nWhen I tried to implement this, I encountered an issue that I don't\nknow how to fix: When we send a partial result containing a direct\nbuffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\nMapped stream/buffer depends on the direct buffer in post-processing,\nthe post-processing might fail due to the lack of the source buffer.\nThis basically stops us from sending buffers back to the application\nearlier than all post-processing done.\n\nTherefore, unless this patch only sets buffers' status and releases\ndirect buffers' fences, it's very hard to be standalone.\n\nWDYT?\n\nBR,\nHarvey\n\n\n>\n> Yes, if we keep the processing (or at least the jpeg one) in\n> requestComplete, then we won't drop the feature.\n>\n> >\n> > 3) Schedule post-processing as soon as the source buffer is available\n> >\n> > Track if for each request the required metadata to populate the EXIF\n> > tags are available (for JPEG) and schedule post-processing as soon\n> > as the source buffer (and metadata for JPEG) are available.\n> >\n> > This indeed might require multiple patches to implement tracking of\n> > the metadata and buffer status\n> >\n> > All steps should be validated with a pipeline that support\n> > metadataAvailable and with a pipeline that doesn't.\n> >\n> > As far as I can all of this currently happens in a single patch (9/9\n> > and partially in 8/9). I might be surely missing details and\n> > implementation issues, but logically the above seems to me a\n> > reasonable break-down of 8/9 and 9/9 ?\n>\n> Yes, I'll try to implement this. Thanks!\n>\n> BR,\n> Harvey\n>\n> >\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > >\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > > >\n> > > > > >\n> > > > > > > >\n> > > > > > > >\n> > > > > > > >\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > WDYT?\n> > > > > > > > > >\n> > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > >\n> > > > > > > >\n> > > > > > > > I might have missed why this is related :)\n> > > > > > >\n> > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > might not have been completed.\n> > > > > > >\n> > > > > > > BR,\n> > > > > > > Harvey\n> > > > > > >\n> > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > >\n> > > > > > > > > > > Thanks\n> > > > > > > > > > > j\n> > > > > > > > > > >\n> > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > >\n> > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > -\n> > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > >\n> > > > > > > > > > > > --\n> > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4884EBD80A\n\tfor ;\n\tThu, 12 Dec 2024 09:00:39 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B53C767EBE;\n\tThu, 12 Dec 2024 10:00:37 +0100 (CET)","from mail-lj1-x230.google.com (mail-lj1-x230.google.com\n\t[IPv6:2a00:1450:4864:20::230])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CEFF167E6D\n\tfor ;\n\tThu, 12 Dec 2024 10:00:35 +0100 (CET)","by mail-lj1-x230.google.com with SMTP id\n\t38308e7fff4ca-30037784fceso2871631fa.2\n\tfor ;\n\tThu, 12 Dec 2024 01:00:35 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"UKr13L+m\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1733994035; x=1734598835;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=7NEtQb1AkvE3KTfDiueTQf405qDuzuFPBhIpmDaWJlw=;\n\tb=UKr13L+msJUBBHxw3C/bUKi1B2whSwrFpaFbQ66G5m/+oTDzbsDk0E3g2ur5tbYIDV\n\thU8lQpTcqR25hShlokUrm02shPP6KWHwL1o+MkviQn4Ust/a5TmvH8YyTA2JaCQ6v1X9\n\tqgZwh5bRW0tmA8lybzs0WJc4mSIOdrQmMJfvs=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1733994035; x=1734598835;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=7NEtQb1AkvE3KTfDiueTQf405qDuzuFPBhIpmDaWJlw=;\n\tb=iOn6NVz/6Zb+W434fjIwyFFZE8vG7oU+SGnxC3a62iWBBwyQ26QPNb7RunQSp96d5d\n\txUxxEv+ZvHqm3d184a4dKik1YQFXhNP+dJ53E1inblOksYq/pC2PDzhSFepSSwUmHqWR\n\tVNGPTzD72S/Frt9e1VV8na6SNpIOJvJMl8FvAVKn0mImOk1miYEKqKU8t9DCXRmfBrvc\n\t1Y0eYMbD2tDI1JaUeSLV0vKadrhsDqAHDHp17yG/hCk0+qaCIdKeEYDrDTpv+FJ6+Tqk\n\truCV73wGPd8RMQTKq47A1VZoxs5S9m3IhoYL6h7K2Tal9m624DqniPsw+NfIi5dMatJr\n\tVEjA==","X-Gm-Message-State":"AOJu0Yx7MwJNHLSjc2qu4ujz8d7+xXCluupJ2K9wF7WZadTLeTaUo1aF\n\twF0aeN+YHIM6QZhdqUWCIeU6GkH/Rp77aVF6tNc7Z2jiBQhy266Xu2NFsYLT7Nwxc+SP2IZ6Dyj\n\tpL6mP13wdoqSeTmSnF+2t35sw8qZW4SBSHbw21zM8YaolL9I=","X-Gm-Gg":"ASbGncvQELfVww/Ii942Jj2qlItkkYgewivrz+U7DAsP1rUzFa7ZGznIHhrvKedHhVk\n\taKfCOMKRADJiKDCgR4Wf+HFynaLiEfwkj+MC9VCFC1F5jfR+lQBA9QDhZ+ib+4Pk8vw==","X-Google-Smtp-Source":"AGHT+IG1dqK8O1QtajL9E9UhVasEVIwAhtsTi4tGaCmP2JitRJLLPE3BsW/yuPghHaSCd+e04Q5PSw9iyQJtz+Y6G38=","X-Received":"by 2002:a2e:bc0d:0:b0:302:1dd5:6b34 with SMTP id\n\t38308e7fff4ca-3024a29b58fmr8100601fa.29.1733994034653;\n\tThu, 12 Dec 2024 01:00:34 -0800 (PST)","MIME-Version":"1.0","References":"<20241127092632.3145984-9-chenghaoyang@chromium.org>\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Thu, 12 Dec 2024 17:00:23 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32711,"web_url":"https://patchwork.libcamera.org/comment/32711/","msgid":"<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>","date":"2024-12-13T08:26:04","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Thu, Dec 12, 2024 at 05:00:23PM +0800, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n> wrote:\n> >\n> > Hi Jacopo,\n> >\n> > On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > > wrote:\n> > > > >\n> > > > > Hi Harvey\n> > > > >\n> > > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Harvey\n> > > > > > >\n> > > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > > wrote:\n> > > > > > > > >\n> > > > > > > > > Hi Harvey\n> > > > > > > > >\n> > > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > Hi Jacopo,\n> > > > > > > > > >\n> > > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > > wrote:\n> > > > > > > > > > >\n> > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > >\n> > > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > > wrote:\n> > > > > > > > > > > >\n> > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > >\n> > > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > > >\n> > > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > > ---\n> > > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > > + }\n> > > > > > > > > > > > > +\n> > > > > > > > > > > >\n> > > > > > > > > > > > no {} for single line statements\n> > > > > > > > > >\n> > > > > > > > > > Done\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > > }\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > +/*\n> > > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > > + */\n> > > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > > +{\n> > > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > > +\n> > > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > > +\n> > > > > > > > > > > > > + /*\n> > > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > > >\n> > > > > > > > > > > > \\todo\n> > > > > > > > > >\n> > > > > > > > > > done\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > > >\n> > > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > > >\n> > > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > > >\n> > > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > > Removed.\n> > > > > > > > >\n> > > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > > keeping a \\todo\n> > > > > > > >\n> > > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > > AnalogueGain directly :)\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > > following patches.\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > > + */\n> > > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > > +}\n> > > > > > > > > > > > > +\n> > > > > > > > > > > > > /*\n> > > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > > */\n> > > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > > + };\n> > > > > > > > > > > > > +\n> > > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > private:\n> > > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > > + {\n> > > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > > + }\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > int configure();\n> > > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > > >\n> > > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > > >\n> > > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > > >\n> > > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > > >\n> > > > > > > > Sure :)\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > BR,\n> > > > > > > > > > Harvey\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > */\n> > > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > > +\n> > > > > > > > > > > >\n> > > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > > >\n> > > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > > threading issue properly.\n> > > > > > > > > > >\n> > > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > > might set further metadata tags.\n> > > > > > > > >\n> > > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > > and buffer completes ?\n> > > > > > > >\n> > > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > > >\n> > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > > >\n> > > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > > process() on it\n> > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > > >\n> > > > > > > > > Is this right ?\n> > > > > > > > >\n> > > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > > >\n> > > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > > >\n> > > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > > >\n> > > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > > the time the buffer to process has completed ?\n> > > > > > > >\n> > > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > > available?\n> > > > > > > >\n> > > > > > >\n> > > > > > > Good question..\n> > > > > > >\n> > > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > > facts run at requestComplete() time.\n> > > > > > >\n> > > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > > produce the jpeg frame as soon as:\n> > > > > > >\n> > > > > > > 1) The source buffer is ready\n> > > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > > your generateJpegExifMetadata()) are available\n> > > > > > >\n> > > > > > > without waiting for the whole request to complete ?\n> > > > > > >\n> > > > > > > In your last patch you introduce support for handling\n> > > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > > happens when the buffer is completed without verifying that the\n> > > > > > > required metadata to populate the EXIF tags are available.\n> > > > > > >\n> > > > > > > Does this match your understanding ?\n> > > > > >\n> > > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > > >\n> > > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > > completed, so perhaps we could expect them to be notified with signal\n> > > > >\n> > > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > > support early metadata completion but could signl buffer being\n> > > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > > metadata before buffers and implement compliance tests to validate\n> > > > > that, I don't think we can assume anything about ordering.\n> > > > >\n> > > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > > you wouldn't like it(?\n> > > > > >\n> > > > >\n> > > > > I don't think signalling metadata at requestComplete time -after-\n> > > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > > what they think about this.\n> > > > >\n> > > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > One possible way to handle this is to\n> > > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > > otherwise skip it\n> > > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > > walk the list of completed buffers to process and run post-processing\n> > > > > >\n> > > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > > >\n> > > > > > >\n> > > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > > general outline ?\n> > > > > > >\n> > > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > > >\n> > > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > > multiple Mapped stream on one Direct.\n> > > > > > >\n> > > > > > > I would take from this series:\n> > > > > > > 2/9\n> > > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > > >\n> > > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > > quite some time breaking this into two though in the beginning haha...\n> > > > > >\n> > > > >\n> > > > > Sorry about that, but the patch is really about supporting multiple\n> > > > > mapped streams, isn't it ?\n> > > >\n> > > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > > supporting multiple mapped streams...\n> > >\n> > > Do you mean 3/9, right ?\n> > >\n> > > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > > about \"supporting multiple mapped streams\"\n> >\n> > Ah okay, thanks :)\n> >\n> > >\n> > > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> > >\n> > > Careful this patch contains a bunch of unrelated refactories at the\n> > > end\n> >\n> > Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> > remove it when uploading :)\n> >\n> > >\n> > > >\n> > > > No worries, I can still squash it.\n> > > >\n> > > > >\n> > > > > > > 5/9\n> > > > > > > 6/9\n> > > > > >\n> > > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > > >\n> > > > >\n> > > > > Thanks\n> > > >\n> > > > We still have some ongoing discussion in the patch. Please help take a\n> > > > look when you have time. Thanks!\n> > >\n> > > On 5/9 and 6/9 ?\n> > >\n> > > 5/9 I sent my R-b tag\n> > > 6/9 I suggested a new commit message but the rest is ok with me\n> >\n> > Yeah on 6/9 I assumed you would have questions about introducing\n> > CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n> >\n> > >\n> > > >\n> > > > >\n> > > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > > >\n> > > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > > >\n> > > > > > Sure :)\n> > > > > >\n> > > > > > > support for handling bufferCompleted() (without handling\n> > > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > > metadataAvailable on top.\n> > > > > >\n> > > > > > I suggest to support signal metadataAvailable first, because:\n> > > > > >\n> > > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > > blocked by post processing, and we could immediately notify the\n> > > > > > application with a partial result.\n> > >\n> > > ack\n> > >\n> > > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > > Android adapter.\n> > > > > > - I've actually done this split in this order, but haven't sent them\n> > >\n> > > fine with this ordering\n> > >\n> > > > > > as patches yet. You can check on gitlab first:\n> > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > > >\n> > > > >\n> > > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > > >\n> > > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > > that doesn't support it yet.\n> > > >\n> > >\n> > > That's great, thanks\n> > >\n> > > > >\n> > > > > > >\n> > > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > > others. What do you think ?\n> > > > > >\n> > > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > > metadata patch into the rest two of the upcoming patches\n> > > > >\n> > > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > > might require more than just two patches :)\n> > > >\n> > > > I really hope I can split them into pieces, while I'm running out of\n> > > > ideas... If you can take a brief look and let me know how you expect\n> > > > them to be split, I'd appreciate that a lot!\n> > > >\n> > > > Some random ideas:\n> > > > On `android: Support partial results with metadataAvailable`:\n> > > >\n> > > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > > the only result.\n> > > > - Add `Camera3ResultDescriptor` and\n> > > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > > migrated), while I'm not sure if it makes sense...\n> > > >\n> > >\n> > > see below\n> > >\n> > > > >\n> > > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > > >\n> > > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > > streams as soon as they're available\" should become quite natural to\n> > > > > be done on top.\n> > > > >\n> > > > > I presume you will need Camera3ResultDescriptor already when\n> > > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > > >\n> > > > Yes, what I meant is that I find it difficult to keep this patch\n> > > > separated from the upcoming patches, unless we implement it on top of\n> > > > them. In this way though, getting the correct result metadata when\n> > >\n> > > Indeed I think it should happen after support for partial metadata and\n> > > bufferCompleted have been developed\n> > >\n> > > > post-processing with jpeg will be unsupported in the middle. Is that\n> > >\n> > > Why do you think so ? As long as process() happens at requestCompleted\n> > > time, all the metadata info you need will be available in\n> > > Request::metadata\n> > >\n> > > > acceptable?\n> > >\n> > > I still fail to see why doing this in lock-step would bring issues,\n> > > but I admit I only glanced through\n> > > 8 files changed, 621 insertions(+), 277 deletions(-)\n> > >\n> > > Could you please tell me in which of the following steps you'll have\n> > > issues:\n> > >\n> > > 1) Add support for metadataAvailable signal\n> > >\n> > > Support the new signal and allow partial metadata completion. The\n> > > HAL can now call process_capture_results multiple times for the\n> > > same request.\n> > >\n> > > Post-processing is not changed as it happens at requestCompleted\n> > > time, when the buffers are available and the required metadata to\n> > > populate EXIF will be available in Request::metadata() as it used\n> > > to be\n> > >\n> > > 2) Add support for bufferCompleted\n> > >\n> > > Support earlier buffer completion. The HAL should already be\n> > > instrumented to send partial results for the Direct buffer that has\n> > > just completed.\n>\n> I assume this includes sending the direct buffers back to the\n> application when receiving signals of bufferCompleted, while delaying\n> post-processing to requestCompleted.\n>\n\nWhat makes you think we have to delay post-processing to\nrequestComplete ?\n\nMy first thought was that as soon as a Direct buffer is ready, if\nany stream is Mapped on it, we can schedule post-processing.\n\n> When I tried to implement this, I encountered an issue that I don't\n> know how to fix: When we send a partial result containing a direct\n> buffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\n\nYeah, you're now giving it back to the Android framework which is\nprobably free to re-use it as it likes.\n\n> Mapped stream/buffer depends on the direct buffer in post-processing,\n> the post-processing might fail due to the lack of the source buffer.\n> This basically stops us from sending buffers back to the application\n> earlier than all post-processing done.\n\nIf, for some reason I'm now missing, we have to delay post-processing\nto requestCompleted, then Direct buffers on which another Stream is\nMapped on have to be delayed to when post-processing is done, as we do\ntoday. Only Direct buffers with no mapped Stream can be signalled\nearlier.\n\nAs said, anticipating post-processing to when both the source buffer\nand the required metadata are available might be desirable, but I\nguess I'm missing something here.\n\n>\n> Therefore, unless this patch only sets buffers' status and releases\n> direct buffers' fences, it's very hard to be standalone.\n>\n> WDYT?\n>\n> BR,\n> Harvey\n>\n>\n> >\n> > Yes, if we keep the processing (or at least the jpeg one) in\n> > requestComplete, then we won't drop the feature.\n> >\n> > >\n> > > 3) Schedule post-processing as soon as the source buffer is available\n> > >\n> > > Track if for each request the required metadata to populate the EXIF\n> > > tags are available (for JPEG) and schedule post-processing as soon\n> > > as the source buffer (and metadata for JPEG) are available.\n> > >\n> > > This indeed might require multiple patches to implement tracking of\n> > > the metadata and buffer status\n> > >\n> > > All steps should be validated with a pipeline that support\n> > > metadataAvailable and with a pipeline that doesn't.\n> > >\n> > > As far as I can all of this currently happens in a single patch (9/9\n> > > and partially in 8/9). I might be surely missing details and\n> > > implementation issues, but logically the above seems to me a\n> > > reasonable break-down of 8/9 and 9/9 ?\n> >\n> > Yes, I'll try to implement this. Thanks!\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > > >\n> > > > > >\n> > > > > > BR,\n> > > > > > Harvey\n> > > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > WDYT?\n> > > > > > > > > > >\n> > > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > I might have missed why this is related :)\n> > > > > > > >\n> > > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > > might not have been completed.\n> > > > > > > >\n> > > > > > > > BR,\n> > > > > > > > Harvey\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > Thanks\n> > > > > > > > > > > > j\n> > > > > > > > > > > >\n> > > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > > -\n> > > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > --\n> > > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id B6722C32F0\n\tfor ;\n\tFri, 13 Dec 2024 08:26:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1D01867EE0;\n\tFri, 13 Dec 2024 09:26:12 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 07407618AD\n\tfor ;\n\tFri, 13 Dec 2024 09:26:07 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 66E4C752;\n\tFri, 13 Dec 2024 09:25:33 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"SLSMUdUX\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1734078333;\n\tbh=u9zzXKKgt4zDadrSG8+ira79EW0MQ5DAlM9CmVjwiY4=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=SLSMUdUXwFqXDg7HffGkQJfc+xkglORH5O2Wsfj4a03oLiVFdMc/zEpfduFZtzabt\n\tpQ/CiKg64Fn8Cqn+xOQseWQ5QkRieSZaSqLdvIo7HFotyNCZQYxvGQ2lwK5+cjPf2m\n\tcw173MGVwF792Z7cN6rA3k3mhJIHAZ9+IUXSnJww=","Date":"Fri, 13 Dec 2024 09:26:04 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>","References":"\n\t\n\t\n\t\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32713,"web_url":"https://patchwork.libcamera.org/comment/32713/","msgid":"","date":"2024-12-13T08:59:24","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Fri, Dec 13, 2024 at 4:26 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Thu, Dec 12, 2024 at 05:00:23PM +0800, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n> > wrote:\n> > >\n> > > Hi Jacopo,\n> > >\n> > > On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Harvey\n> > > > > >\n> > > > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > > > wrote:\n> > > > > > > >\n> > > > > > > > Hi Harvey\n> > > > > > > >\n> > > > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > Hi Jacopo,\n> > > > > > > > >\n> > > > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > > > wrote:\n> > > > > > > > > >\n> > > > > > > > > > Hi Harvey\n> > > > > > > > > >\n> > > > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > >\n> > > > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > > > wrote:\n> > > > > > > > > > > >\n> > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > >\n> > > > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > > > wrote:\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > > > ---\n> > > > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > no {} for single line statements\n> > > > > > > > > > >\n> > > > > > > > > > > Done\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > > > }\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > +/*\n> > > > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > > > +{\n> > > > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > + /*\n> > > > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > \\todo\n> > > > > > > > > > >\n> > > > > > > > > > > done\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > > > >\n> > > > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > > > Removed.\n> > > > > > > > > >\n> > > > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > > > keeping a \\todo\n> > > > > > > > >\n> > > > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > > > AnalogueGain directly :)\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > > > following patches.\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > > > +}\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > /*\n> > > > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > > > + };\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > private:\n> > > > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > > > + {\n> > > > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > int configure();\n> > > > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > > > >\n> > > > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > > > >\n> > > > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > > > >\n> > > > > > > > > Sure :)\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > BR,\n> > > > > > > > > > > Harvey\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > > > +\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > > > >\n> > > > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > > > threading issue properly.\n> > > > > > > > > > > >\n> > > > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > > > might set further metadata tags.\n> > > > > > > > > >\n> > > > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > > > and buffer completes ?\n> > > > > > > > >\n> > > > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > > > >\n> > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > > > >\n> > > > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > > > process() on it\n> > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > > > >\n> > > > > > > > > > Is this right ?\n> > > > > > > > > >\n> > > > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > > > >\n> > > > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > > > >\n> > > > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > > > >\n> > > > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > > > the time the buffer to process has completed ?\n> > > > > > > > >\n> > > > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > > > available?\n> > > > > > > > >\n> > > > > > > >\n> > > > > > > > Good question..\n> > > > > > > >\n> > > > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > > > facts run at requestComplete() time.\n> > > > > > > >\n> > > > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > > > produce the jpeg frame as soon as:\n> > > > > > > >\n> > > > > > > > 1) The source buffer is ready\n> > > > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > > > your generateJpegExifMetadata()) are available\n> > > > > > > >\n> > > > > > > > without waiting for the whole request to complete ?\n> > > > > > > >\n> > > > > > > > In your last patch you introduce support for handling\n> > > > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > > > happens when the buffer is completed without verifying that the\n> > > > > > > > required metadata to populate the EXIF tags are available.\n> > > > > > > >\n> > > > > > > > Does this match your understanding ?\n> > > > > > >\n> > > > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > > > >\n> > > > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > > > completed, so perhaps we could expect them to be notified with signal\n> > > > > >\n> > > > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > > > support early metadata completion but could signl buffer being\n> > > > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > > > metadata before buffers and implement compliance tests to validate\n> > > > > > that, I don't think we can assume anything about ordering.\n> > > > > >\n> > > > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > > > you wouldn't like it(?\n> > > > > > >\n> > > > > >\n> > > > > > I don't think signalling metadata at requestComplete time -after-\n> > > > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > > > what they think about this.\n> > > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > One possible way to handle this is to\n> > > > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > > > otherwise skip it\n> > > > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > > > walk the list of completed buffers to process and run post-processing\n> > > > > > >\n> > > > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > > > >\n> > > > > > > >\n> > > > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > > > general outline ?\n> > > > > > > >\n> > > > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > > > >\n> > > > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > > > multiple Mapped stream on one Direct.\n> > > > > > > >\n> > > > > > > > I would take from this series:\n> > > > > > > > 2/9\n> > > > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > > > >\n> > > > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > > > quite some time breaking this into two though in the beginning haha...\n> > > > > > >\n> > > > > >\n> > > > > > Sorry about that, but the patch is really about supporting multiple\n> > > > > > mapped streams, isn't it ?\n> > > > >\n> > > > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > > > supporting multiple mapped streams...\n> > > >\n> > > > Do you mean 3/9, right ?\n> > > >\n> > > > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > > > about \"supporting multiple mapped streams\"\n> > >\n> > > Ah okay, thanks :)\n> > >\n> > > >\n> > > > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> > > >\n> > > > Careful this patch contains a bunch of unrelated refactories at the\n> > > > end\n> > >\n> > > Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> > > remove it when uploading :)\n> > >\n> > > >\n> > > > >\n> > > > > No worries, I can still squash it.\n> > > > >\n> > > > > >\n> > > > > > > > 5/9\n> > > > > > > > 6/9\n> > > > > > >\n> > > > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > > > >\n> > > > > >\n> > > > > > Thanks\n> > > > >\n> > > > > We still have some ongoing discussion in the patch. Please help take a\n> > > > > look when you have time. Thanks!\n> > > >\n> > > > On 5/9 and 6/9 ?\n> > > >\n> > > > 5/9 I sent my R-b tag\n> > > > 6/9 I suggested a new commit message but the rest is ok with me\n> > >\n> > > Yeah on 6/9 I assumed you would have questions about introducing\n> > > CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n> > >\n> > > >\n> > > > >\n> > > > > >\n> > > > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > > > >\n> > > > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > > > >\n> > > > > > > Sure :)\n> > > > > > >\n> > > > > > > > support for handling bufferCompleted() (without handling\n> > > > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > > > metadataAvailable on top.\n> > > > > > >\n> > > > > > > I suggest to support signal metadataAvailable first, because:\n> > > > > > >\n> > > > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > > > blocked by post processing, and we could immediately notify the\n> > > > > > > application with a partial result.\n> > > >\n> > > > ack\n> > > >\n> > > > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > > > Android adapter.\n> > > > > > > - I've actually done this split in this order, but haven't sent them\n> > > >\n> > > > fine with this ordering\n> > > >\n> > > > > > > as patches yet. You can check on gitlab first:\n> > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > > > >\n> > > > > >\n> > > > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > > > >\n> > > > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > > > that doesn't support it yet.\n> > > > >\n> > > >\n> > > > That's great, thanks\n> > > >\n> > > > > >\n> > > > > > > >\n> > > > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > > > others. What do you think ?\n> > > > > > >\n> > > > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > > > metadata patch into the rest two of the upcoming patches\n> > > > > >\n> > > > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > > > might require more than just two patches :)\n> > > > >\n> > > > > I really hope I can split them into pieces, while I'm running out of\n> > > > > ideas... If you can take a brief look and let me know how you expect\n> > > > > them to be split, I'd appreciate that a lot!\n> > > > >\n> > > > > Some random ideas:\n> > > > > On `android: Support partial results with metadataAvailable`:\n> > > > >\n> > > > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > > > the only result.\n> > > > > - Add `Camera3ResultDescriptor` and\n> > > > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > > > migrated), while I'm not sure if it makes sense...\n> > > > >\n> > > >\n> > > > see below\n> > > >\n> > > > > >\n> > > > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > > > >\n> > > > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > > > streams as soon as they're available\" should become quite natural to\n> > > > > > be done on top.\n> > > > > >\n> > > > > > I presume you will need Camera3ResultDescriptor already when\n> > > > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > > > >\n> > > > > Yes, what I meant is that I find it difficult to keep this patch\n> > > > > separated from the upcoming patches, unless we implement it on top of\n> > > > > them. In this way though, getting the correct result metadata when\n> > > >\n> > > > Indeed I think it should happen after support for partial metadata and\n> > > > bufferCompleted have been developed\n> > > >\n> > > > > post-processing with jpeg will be unsupported in the middle. Is that\n> > > >\n> > > > Why do you think so ? As long as process() happens at requestCompleted\n> > > > time, all the metadata info you need will be available in\n> > > > Request::metadata\n> > > >\n> > > > > acceptable?\n> > > >\n> > > > I still fail to see why doing this in lock-step would bring issues,\n> > > > but I admit I only glanced through\n> > > > 8 files changed, 621 insertions(+), 277 deletions(-)\n> > > >\n> > > > Could you please tell me in which of the following steps you'll have\n> > > > issues:\n> > > >\n> > > > 1) Add support for metadataAvailable signal\n> > > >\n> > > > Support the new signal and allow partial metadata completion. The\n> > > > HAL can now call process_capture_results multiple times for the\n> > > > same request.\n> > > >\n> > > > Post-processing is not changed as it happens at requestCompleted\n> > > > time, when the buffers are available and the required metadata to\n> > > > populate EXIF will be available in Request::metadata() as it used\n> > > > to be\n> > > >\n> > > > 2) Add support for bufferCompleted\n> > > >\n> > > > Support earlier buffer completion. The HAL should already be\n> > > > instrumented to send partial results for the Direct buffer that has\n> > > > just completed.\n> >\n> > I assume this includes sending the direct buffers back to the\n> > application when receiving signals of bufferCompleted, while delaying\n> > post-processing to requestCompleted.\n> >\n>\n> What makes you think we have to delay post-processing to\n> requestComplete ?\n>\n> My first thought was that as soon as a Direct buffer is ready, if\n> any stream is Mapped on it, we can schedule post-processing.\n\nI assumed that's in the third step (patch). I don't know how to start\npost-processing without the functionalities to check if the required\nmetadata to populate the EXIF tags are available (for JPEG).\n\n>\n> > When I tried to implement this, I encountered an issue that I don't\n> > know how to fix: When we send a partial result containing a direct\n> > buffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\n>\n> Yeah, you're now giving it back to the Android framework which is\n> probably free to re-use it as it likes.\n>\n> > Mapped stream/buffer depends on the direct buffer in post-processing,\n> > the post-processing might fail due to the lack of the source buffer.\n> > This basically stops us from sending buffers back to the application\n> > earlier than all post-processing done.\n>\n> If, for some reason I'm now missing, we have to delay post-processing\n> to requestCompleted, then Direct buffers on which another Stream is\n> Mapped on have to be delayed to when post-processing is done, as we do\n> today. Only Direct buffers with no mapped Stream can be signalled\n> earlier.\n>\n> As said, anticipating post-processing to when both the source buffer\n> and the required metadata are available might be desirable, but I\n> guess I'm missing something here.\n\nExactly, checking the required metadata is mandatory to do\npost-processing earlier in partial results.\nIn our previous plan, it happens after the patch to introduce the\nbufferCompleted signal.\n\nBR,\nHarvey\n\n>\n> >\n> > Therefore, unless this patch only sets buffers' status and releases\n> > direct buffers' fences, it's very hard to be standalone.\n> >\n> > WDYT?\n> >\n> > BR,\n> > Harvey\n> >\n> >\n> > >\n> > > Yes, if we keep the processing (or at least the jpeg one) in\n> > > requestComplete, then we won't drop the feature.\n> > >\n> > > >\n> > > > 3) Schedule post-processing as soon as the source buffer is available\n> > > >\n> > > > Track if for each request the required metadata to populate the EXIF\n> > > > tags are available (for JPEG) and schedule post-processing as soon\n> > > > as the source buffer (and metadata for JPEG) are available.\n> > > >\n> > > > This indeed might require multiple patches to implement tracking of\n> > > > the metadata and buffer status\n> > > >\n> > > > All steps should be validated with a pipeline that support\n> > > > metadataAvailable and with a pipeline that doesn't.\n> > > >\n> > > > As far as I can all of this currently happens in a single patch (9/9\n> > > > and partially in 8/9). I might be surely missing details and\n> > > > implementation issues, but logically the above seems to me a\n> > > > reasonable break-down of 8/9 and 9/9 ?\n> > >\n> > > Yes, I'll try to implement this. Thanks!\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > >\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > BR,\n> > > > > > > Harvey\n> > > > > > >\n> > > > > > > >\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > WDYT?\n> > > > > > > > > > > >\n> > > > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > I might have missed why this is related :)\n> > > > > > > > >\n> > > > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > > > might not have been completed.\n> > > > > > > > >\n> > > > > > > > > BR,\n> > > > > > > > > Harvey\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Thanks\n> > > > > > > > > > > > > j\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > > > -\n> > > > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > --\n> > > > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id B7D86C32F0\n\tfor ;\n\tFri, 13 Dec 2024 08:59:42 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7CD2967EE4;\n\tFri, 13 Dec 2024 09:59:41 +0100 (CET)","from mail-lj1-x232.google.com (mail-lj1-x232.google.com\n\t[IPv6:2a00:1450:4864:20::232])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 10448618AD\n\tfor ;\n\tFri, 13 Dec 2024 09:59:38 +0100 (CET)","by mail-lj1-x232.google.com with SMTP id\n\t38308e7fff4ca-3023c51146cso14856801fa.1\n\tfor ;\n\tFri, 13 Dec 2024 00:59:37 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"jyOT8Thx\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1734080377; x=1734685177;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=y/tUM2/N0DddenrRMJW4QuVd+uoQ6cwjJ0+G8cUx7Eo=;\n\tb=jyOT8ThxQcjWvsOEPKBFRA1P7QzQ8PbEmzqYKoBrNHLwos9MaHhhO0wFXK0FCt4skG\n\tu62C1wPT4LNc5OHgtXj7QX1USpTbuUm5EpMrOHViO5j9axicOWviTiIsE2S3TQnqdA61\n\tWSzAMr/98l0/qQkAj/AnsB3jrG27opzjbO2Pc=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1734080377; x=1734685177;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=y/tUM2/N0DddenrRMJW4QuVd+uoQ6cwjJ0+G8cUx7Eo=;\n\tb=Aia/yd7QLY/LpZYU21fRd58570K6WgTt1Na91EVt6wThIVqcTXZfCateivBS7sgfl/\n\t5RQK3hnV1kay/Szs4JZeDeu/yeVM6Ha3uJ7cGRz9NXndPQNVG/kifztkaLIIeKaEP+rc\n\twX1LLE6Q20B2QcGsy/vT9gQHpyHQo0eJRyW4HDCXBY2NuvjnEyVmf2tNjLdlAg+zzWVw\n\tmwJKsLb9g5p5eSYAFLm+6vp40ONQxPd8Kq6Qn1UCNTlZ7CkzX6kOT4FF3wsVHFVgTgS8\n\tRhcyZP4I8c/Q9N1v+5Vk5Y21fTyFRjmDFHM/SXcg2Q6zx6dt63taVXzQ0aOXZMnTL2LX\n\tmuFQ==","X-Gm-Message-State":"AOJu0YxIhBm1R1rJNW0pa1JMLMDLjpGASr6EKf3tDFnyTyk1UW4y8pA5\n\tl/sdMowRx7jM6YHRoDxTv7F+LzpKGZUa+HCCf1I3PIAGMhlQRD5MESqw8wDcCOMTdaEA3QwsJld\n\tkMYRNnv3Q3IwOAfUbH1Al+o+FZXkcvrT5ktup","X-Gm-Gg":"ASbGncvXIEr5cq1vW3sNLz61iUetYJaAU2XioQWgMKlFWUn0ARR/H7VuA+Khu3GLe1s\n\tpRHfFoyLdn6D9Lt8tPOlYFj20kMQBSXc0jrAAGOmc0BueXJy6i6ss8dE8zjOMLu19uQ==","X-Google-Smtp-Source":"AGHT+IGVWMkXuoRFIBMjOAqxt6pc0YnESYcHDtTzd0gMEV9d+sKXdUAGAYyavVOP79I6ZIyWbP7UGECmOd+0kcBQukM=","X-Received":"by 2002:a2e:8095:0:b0:2ff:5d45:883d with SMTP id\n\t38308e7fff4ca-302a4d2182bmr3638301fa.13.1734080375295;\n\tFri, 13 Dec 2024 00:59:35 -0800 (PST)","MIME-Version":"1.0","References":"\n\t\n\t\n\t\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t\n\t\n\t<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>","In-Reply-To":"<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>","From":"Cheng-Hao Yang ","Date":"Fri, 13 Dec 2024 16:59:24 +0800","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32913,"web_url":"https://patchwork.libcamera.org/comment/32913/","msgid":"","date":"2025-01-02T13:11:57","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Fri, Dec 13, 2024 at 9:59 AM Cheng-Hao Yang\n wrote:\n>\n> Hi Jacopo,\n>\n> On Fri, Dec 13, 2024 at 4:26 PM Jacopo Mondi\n> wrote:\n> >\n> > Hi Harvey\n> >\n> > On Thu, Dec 12, 2024 at 05:00:23PM +0800, Cheng-Hao Yang wrote:\n> > > Hi Jacopo,\n> > >\n> > > On Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n> > > wrote:\n> > > >\n> > > > Hi Jacopo,\n> > > >\n> > > > On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> > > > wrote:\n> > > > >\n> > > > > Hi Harvey\n> > > > >\n> > > > > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Harvey\n> > > > > > >\n> > > > > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > > > > wrote:\n> > > > > > > > >\n> > > > > > > > > Hi Harvey\n> > > > > > > > >\n> > > > > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > Hi Jacopo,\n> > > > > > > > > >\n> > > > > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > > > > wrote:\n> > > > > > > > > > >\n> > > > > > > > > > > Hi Harvey\n> > > > > > > > > > >\n> > > > > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > >\n> > > > > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > > > > wrote:\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > > > > wrote:\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > > > > ---\n> > > > > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > no {} for single line statements\n> > > > > > > > > > > >\n> > > > > > > > > > > > Done\n> > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > > > > }\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > +/*\n> > > > > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > > > > +{\n> > > > > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > + /*\n> > > > > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > \\todo\n> > > > > > > > > > > >\n> > > > > > > > > > > > done\n> > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > > > > >\n> > > > > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > > > > Removed.\n> > > > > > > > > > >\n> > > > > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > > > > keeping a \\todo\n> > > > > > > > > >\n> > > > > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > > > > AnalogueGain directly :)\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > > > > following patches.\n> > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > > > > +}\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > /*\n> > > > > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > > > > + };\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > private:\n> > > > > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > > > > + {\n> > > > > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > int configure();\n> > > > > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > > > > >\n> > > > > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > > > > >\n> > > > > > > > > > Sure :)\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > BR,\n> > > > > > > > > > > > Harvey\n> > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > > > > threading issue properly.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > > > > might set further metadata tags.\n> > > > > > > > > > >\n> > > > > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > > > > and buffer completes ?\n> > > > > > > > > >\n> > > > > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > > > > >\n> > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > > > > >\n> > > > > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > > > > process() on it\n> > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > > > > >\n> > > > > > > > > > > Is this right ?\n> > > > > > > > > > >\n> > > > > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > > > > >\n> > > > > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > > > > >\n> > > > > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > > > > >\n> > > > > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > > > > the time the buffer to process has completed ?\n> > > > > > > > > >\n> > > > > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > > > > available?\n> > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > Good question..\n> > > > > > > > >\n> > > > > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > > > > facts run at requestComplete() time.\n> > > > > > > > >\n> > > > > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > > > > produce the jpeg frame as soon as:\n> > > > > > > > >\n> > > > > > > > > 1) The source buffer is ready\n> > > > > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > > > > your generateJpegExifMetadata()) are available\n> > > > > > > > >\n> > > > > > > > > without waiting for the whole request to complete ?\n> > > > > > > > >\n> > > > > > > > > In your last patch you introduce support for handling\n> > > > > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > > > > happens when the buffer is completed without verifying that the\n> > > > > > > > > required metadata to populate the EXIF tags are available.\n> > > > > > > > >\n> > > > > > > > > Does this match your understanding ?\n> > > > > > > >\n> > > > > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > > > > >\n> > > > > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > > > > completed, so perhaps we could expect them to be notified with signal\n> > > > > > >\n> > > > > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > > > > support early metadata completion but could signl buffer being\n> > > > > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > > > > metadata before buffers and implement compliance tests to validate\n> > > > > > > that, I don't think we can assume anything about ordering.\n> > > > > > >\n> > > > > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > > > > you wouldn't like it(?\n> > > > > > > >\n> > > > > > >\n> > > > > > > I don't think signalling metadata at requestComplete time -after-\n> > > > > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > > > > what they think about this.\n> > > > > > >\n> > > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > One possible way to handle this is to\n> > > > > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > > > > otherwise skip it\n> > > > > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > > > > walk the list of completed buffers to process and run post-processing\n> > > > > > > >\n> > > > > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > > > > general outline ?\n> > > > > > > > >\n> > > > > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > > > > >\n> > > > > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > > > > multiple Mapped stream on one Direct.\n> > > > > > > > >\n> > > > > > > > > I would take from this series:\n> > > > > > > > > 2/9\n> > > > > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > > > > >\n> > > > > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > > > > quite some time breaking this into two though in the beginning haha...\n> > > > > > > >\n> > > > > > >\n> > > > > > > Sorry about that, but the patch is really about supporting multiple\n> > > > > > > mapped streams, isn't it ?\n> > > > > >\n> > > > > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > > > > supporting multiple mapped streams...\n> > > > >\n> > > > > Do you mean 3/9, right ?\n> > > > >\n> > > > > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > > > > about \"supporting multiple mapped streams\"\n> > > >\n> > > > Ah okay, thanks :)\n> > > >\n> > > > >\n> > > > > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> > > > >\n> > > > > Careful this patch contains a bunch of unrelated refactories at the\n> > > > > end\n> > > >\n> > > > Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> > > > remove it when uploading :)\n> > > >\n> > > > >\n> > > > > >\n> > > > > > No worries, I can still squash it.\n> > > > > >\n> > > > > > >\n> > > > > > > > > 5/9\n> > > > > > > > > 6/9\n> > > > > > > >\n> > > > > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > > > > >\n> > > > > > >\n> > > > > > > Thanks\n> > > > > >\n> > > > > > We still have some ongoing discussion in the patch. Please help take a\n> > > > > > look when you have time. Thanks!\n> > > > >\n> > > > > On 5/9 and 6/9 ?\n> > > > >\n> > > > > 5/9 I sent my R-b tag\n> > > > > 6/9 I suggested a new commit message but the rest is ok with me\n> > > >\n> > > > Yeah on 6/9 I assumed you would have questions about introducing\n> > > > CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n> > > >\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > > > > >\n> > > > > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > > > > >\n> > > > > > > > Sure :)\n> > > > > > > >\n> > > > > > > > > support for handling bufferCompleted() (without handling\n> > > > > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > > > > metadataAvailable on top.\n> > > > > > > >\n> > > > > > > > I suggest to support signal metadataAvailable first, because:\n> > > > > > > >\n> > > > > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > > > > blocked by post processing, and we could immediately notify the\n> > > > > > > > application with a partial result.\n> > > > >\n> > > > > ack\n> > > > >\n> > > > > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > > > > Android adapter.\n> > > > > > > > - I've actually done this split in this order, but haven't sent them\n> > > > >\n> > > > > fine with this ordering\n> > > > >\n> > > > > > > > as patches yet. You can check on gitlab first:\n> > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > > > > >\n> > > > > > >\n> > > > > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > > > > >\n> > > > > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > > > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > > > > that doesn't support it yet.\n> > > > > >\n> > > > >\n> > > > > That's great, thanks\n> > > > >\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > > > > others. What do you think ?\n> > > > > > > >\n> > > > > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > > > > metadata patch into the rest two of the upcoming patches\n> > > > > > >\n> > > > > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > > > > might require more than just two patches :)\n> > > > > >\n> > > > > > I really hope I can split them into pieces, while I'm running out of\n> > > > > > ideas... If you can take a brief look and let me know how you expect\n> > > > > > them to be split, I'd appreciate that a lot!\n> > > > > >\n> > > > > > Some random ideas:\n> > > > > > On `android: Support partial results with metadataAvailable`:\n> > > > > >\n> > > > > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > > > > the only result.\n> > > > > > - Add `Camera3ResultDescriptor` and\n> > > > > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > > > > migrated), while I'm not sure if it makes sense...\n> > > > > >\n> > > > >\n> > > > > see below\n> > > > >\n> > > > > > >\n> > > > > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > > > > >\n> > > > > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > > > > streams as soon as they're available\" should become quite natural to\n> > > > > > > be done on top.\n> > > > > > >\n> > > > > > > I presume you will need Camera3ResultDescriptor already when\n> > > > > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > > > > >\n> > > > > > Yes, what I meant is that I find it difficult to keep this patch\n> > > > > > separated from the upcoming patches, unless we implement it on top of\n> > > > > > them. In this way though, getting the correct result metadata when\n> > > > >\n> > > > > Indeed I think it should happen after support for partial metadata and\n> > > > > bufferCompleted have been developed\n> > > > >\n> > > > > > post-processing with jpeg will be unsupported in the middle. Is that\n> > > > >\n> > > > > Why do you think so ? As long as process() happens at requestCompleted\n> > > > > time, all the metadata info you need will be available in\n> > > > > Request::metadata\n> > > > >\n> > > > > > acceptable?\n> > > > >\n> > > > > I still fail to see why doing this in lock-step would bring issues,\n> > > > > but I admit I only glanced through\n> > > > > 8 files changed, 621 insertions(+), 277 deletions(-)\n> > > > >\n> > > > > Could you please tell me in which of the following steps you'll have\n> > > > > issues:\n> > > > >\n> > > > > 1) Add support for metadataAvailable signal\n> > > > >\n> > > > > Support the new signal and allow partial metadata completion. The\n> > > > > HAL can now call process_capture_results multiple times for the\n> > > > > same request.\n> > > > >\n> > > > > Post-processing is not changed as it happens at requestCompleted\n> > > > > time, when the buffers are available and the required metadata to\n> > > > > populate EXIF will be available in Request::metadata() as it used\n> > > > > to be\n> > > > >\n> > > > > 2) Add support for bufferCompleted\n> > > > >\n> > > > > Support earlier buffer completion. The HAL should already be\n> > > > > instrumented to send partial results for the Direct buffer that has\n> > > > > just completed.\n> > >\n> > > I assume this includes sending the direct buffers back to the\n> > > application when receiving signals of bufferCompleted, while delaying\n> > > post-processing to requestCompleted.\n> > >\n> >\n> > What makes you think we have to delay post-processing to\n> > requestComplete ?\n> >\n> > My first thought was that as soon as a Direct buffer is ready, if\n> > any stream is Mapped on it, we can schedule post-processing.\n>\n> I assumed that's in the third step (patch). I don't know how to start\n> post-processing without the functionalities to check if the required\n> metadata to populate the EXIF tags are available (for JPEG).\n>\n> >\n> > > When I tried to implement this, I encountered an issue that I don't\n> > > know how to fix: When we send a partial result containing a direct\n> > > buffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\n> >\n> > Yeah, you're now giving it back to the Android framework which is\n> > probably free to re-use it as it likes.\n> >\n> > > Mapped stream/buffer depends on the direct buffer in post-processing,\n> > > the post-processing might fail due to the lack of the source buffer.\n> > > This basically stops us from sending buffers back to the application\n> > > earlier than all post-processing done.\n> >\n> > If, for some reason I'm now missing, we have to delay post-processing\n> > to requestCompleted, then Direct buffers on which another Stream is\n> > Mapped on have to be delayed to when post-processing is done, as we do\n> > today. Only Direct buffers with no mapped Stream can be signalled\n> > earlier.\n> >\n> > As said, anticipating post-processing to when both the source buffer\n> > and the required metadata are available might be desirable, but I\n> > guess I'm missing something here.\n>\n> Exactly, checking the required metadata is mandatory to do\n> post-processing earlier in partial results.\n> In our previous plan, it happens after the patch to introduce the\n> bufferCompleted signal.\n\nFriendly ping: any update?\n\nBR,\nHarvey\n\n>\n> BR,\n> Harvey\n>\n> >\n> > >\n> > > Therefore, unless this patch only sets buffers' status and releases\n> > > direct buffers' fences, it's very hard to be standalone.\n> > >\n> > > WDYT?\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > >\n> > > >\n> > > > Yes, if we keep the processing (or at least the jpeg one) in\n> > > > requestComplete, then we won't drop the feature.\n> > > >\n> > > > >\n> > > > > 3) Schedule post-processing as soon as the source buffer is available\n> > > > >\n> > > > > Track if for each request the required metadata to populate the EXIF\n> > > > > tags are available (for JPEG) and schedule post-processing as soon\n> > > > > as the source buffer (and metadata for JPEG) are available.\n> > > > >\n> > > > > This indeed might require multiple patches to implement tracking of\n> > > > > the metadata and buffer status\n> > > > >\n> > > > > All steps should be validated with a pipeline that support\n> > > > > metadataAvailable and with a pipeline that doesn't.\n> > > > >\n> > > > > As far as I can all of this currently happens in a single patch (9/9\n> > > > > and partially in 8/9). I might be surely missing details and\n> > > > > implementation issues, but logically the above seems to me a\n> > > > > reasonable break-down of 8/9 and 9/9 ?\n> > > >\n> > > > Yes, I'll try to implement this. Thanks!\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > > >\n> > > > > >\n> > > > > > BR,\n> > > > > > Harvey\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > BR,\n> > > > > > > > Harvey\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > WDYT?\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > I might have missed why this is related :)\n> > > > > > > > > >\n> > > > > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > > > > might not have been completed.\n> > > > > > > > > >\n> > > > > > > > > > BR,\n> > > > > > > > > > Harvey\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Thanks\n> > > > > > > > > > > > > > j\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > > > > -\n> > > > > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > --\n> > > > > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id CB2CAC32C2\n\tfor ;\n\tThu, 2 Jan 2025 13:12:12 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6712D684D9;\n\tThu, 2 Jan 2025 14:12:11 +0100 (CET)","from mail-lj1-x22f.google.com (mail-lj1-x22f.google.com\n\t[IPv6:2a00:1450:4864:20::22f])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BF8BD684CC\n\tfor ;\n\tThu, 2 Jan 2025 14:12:09 +0100 (CET)","by mail-lj1-x22f.google.com with SMTP id\n\t38308e7fff4ca-304d760f118so3983021fa.0\n\tfor ;\n\tThu, 02 Jan 2025 05:12:09 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"Dc94XVBd\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1735823529; x=1736428329;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=ZhhBLlkku0jONAJWNsO7x+wm5YpA0pklDC9H9W2VRR0=;\n\tb=Dc94XVBdJLrMzYeBIIj62mCWoH4O6wF4JHe/rXqa9ZuLjXNTaxsF/94BGklwLjwrau\n\twLw+2Df9XD6e3A/cmijgwciIIat8NZnCkxeBb6LZNpyNkR4b8RzoJuSyusyhjEOSZIHs\n\tJ9GbAL2aJ6LPgF8teDDu5J6Jqvc8DdAdSZzA0=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1735823529; x=1736428329;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=ZhhBLlkku0jONAJWNsO7x+wm5YpA0pklDC9H9W2VRR0=;\n\tb=uNzBKR0qu1A50HXOepJBzywrMVYB3mtJkncPJURL97tDaHd8KB1fXZwwgDGRXIaeft\n\tCmgnWYEISBesJnd2fRKh4TU3yJpTlyOhhkksTWCWOt2IrkoR5nSvA9J6+z4dlQ4On5mq\n\tTTFWKg8i1j/5a8EwBIxYSV2zgrpTDYg+mDC3l1NL5Q3lXxHE5FdJIcdKa1RFQELW4jiL\n\tCwvPwf+6jYkR7Dj52lWsj0b2/knd3GYCg+LbKV9vwO1+hPjriCsuNDOYYTS2d2qQttgF\n\t2dx4DpwycQiRTO4ZrG22oOnFuky704bKl0X3dey623MobcaiHBDvPRKibA9OKBXmzJAV\n\tKBRA==","X-Gm-Message-State":"AOJu0YwyjFAdtfrI562dUpIoiKUtuwlJKeGc1wzyjs847dONncBnFTUC\n\tfnsWd5iLwfJEC2UG81+4ebUaRf6PwCXvCwBxnELZ8lT3Y1YaO1C5ID+2XsYOkSoPQfMGE4/AdUW\n\tpekSBT+eN2r4RBQKIS/IuOpQKQJypnHexeU1w","X-Gm-Gg":"ASbGncvA+HVxE3ygAsdip0+U1irjkLhyC8vivcNzOW1eryoxrOE+nC+j4Q/01RAgk3y\n\tk3X/mv/2aWNlIsn6ZDAkPzlZPri5TjlStGzcB","X-Google-Smtp-Source":"AGHT+IFpoZgh0Pv4qweLy3MqELjc2F2lAtArpJJPAaeiaCH+bniu+6OHmsdmbPJdbsRA5BstwQyYGLed8J6G5PyICgs=","X-Received":"by 2002:a2e:b8d5:0:b0:300:2464:c0c2 with SMTP id\n\t38308e7fff4ca-3046850a101mr142840411fa.8.1735823528572;\n\tThu, 02 Jan 2025 05:12:08 -0800 (PST)","MIME-Version":"1.0","References":"\n\t\n\t\n\t\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t\n\t\n\t<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Thu, 2 Jan 2025 14:11:57 +0100","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32916,"web_url":"https://patchwork.libcamera.org/comment/32916/","msgid":"","date":"2025-01-02T15:26:19","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Harvey\n\n to me it's clear this thread has reached a length which makes it\nrather hard to continue the discussion.\n\nAs I recall the status was that you have broken out\n[PATCH v4 0/7] Refactor Android HAL before supporting partial result\n\nfrom\n[PATCH v2 0/8] Signal metadataAvailable and Android partial result\n\nand we agreed we should have found a better solution for partial\nresult support in libcamera before handling it in Android.\n\nFor this purpose I sent on the 6th December\n[PATCH 0/8] libcamera: Support partial metadata completion\nhttps://patchwork.libcamera.org/project/libcamera/list/?series=4858\non which I have cc-ed you and Han-lin\n\nThe series adds support for partial results to libcamera, and the\nandroid handling should be designed on top of this (or any other\nsolution that we decide is the correct one).\n\nI haven't received feedback on that series, have you perhaps missed it ?\n\n\n\nOn Thu, Jan 02, 2025 at 02:11:57PM +0100, Cheng-Hao Yang wrote:\n> Hi Jacopo,\n>\n> On Fri, Dec 13, 2024 at 9:59 AM Cheng-Hao Yang\n> wrote:\n> >\n> > Hi Jacopo,\n> >\n> > On Fri, Dec 13, 2024 at 4:26 PM Jacopo Mondi\n> > wrote:\n> > >\n> > > Hi Harvey\n> > >\n> > > On Thu, Dec 12, 2024 at 05:00:23PM +0800, Cheng-Hao Yang wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n> > > > wrote:\n> > > > >\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Harvey\n> > > > > >\n> > > > > > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > > > > > wrote:\n> > > > > > > >\n> > > > > > > > Hi Harvey\n> > > > > > > >\n> > > > > > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > Hi Jacopo,\n> > > > > > > > >\n> > > > > > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > > > > > wrote:\n> > > > > > > > > >\n> > > > > > > > > > Hi Harvey\n> > > > > > > > > >\n> > > > > > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > >\n> > > > > > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > > > > > wrote:\n> > > > > > > > > > > >\n> > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > >\n> > > > > > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > > > > > wrote:\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > > > > > wrote:\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > > > > > ---\n> > > > > > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > no {} for single line statements\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Done\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > > > > > }\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > +/*\n> > > > > > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > > > > > +{\n> > > > > > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > + /*\n> > > > > > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > \\todo\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > done\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > > > > > Removed.\n> > > > > > > > > > > >\n> > > > > > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > > > > > keeping a \\todo\n> > > > > > > > > > >\n> > > > > > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > > > > > AnalogueGain directly :)\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > > > > > following patches.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > > > > > +}\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > /*\n> > > > > > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > > > > > + };\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > private:\n> > > > > > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > > > > > + {\n> > > > > > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > int configure();\n> > > > > > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > > > > > >\n> > > > > > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > > > > > >\n> > > > > > > > > > > Sure :)\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > BR,\n> > > > > > > > > > > > > Harvey\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > > > > > threading issue properly.\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > > > > > might set further metadata tags.\n> > > > > > > > > > > >\n> > > > > > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > > > > > and buffer completes ?\n> > > > > > > > > > >\n> > > > > > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > > > > > >\n> > > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > > > > > >\n> > > > > > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > > > > > process() on it\n> > > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > > > > > >\n> > > > > > > > > > > > Is this right ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > > > > > >\n> > > > > > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > > > > > >\n> > > > > > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > > > > > >\n> > > > > > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > > > > > the time the buffer to process has completed ?\n> > > > > > > > > > >\n> > > > > > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > > > > > available?\n> > > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > Good question..\n> > > > > > > > > >\n> > > > > > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > > > > > facts run at requestComplete() time.\n> > > > > > > > > >\n> > > > > > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > > > > > produce the jpeg frame as soon as:\n> > > > > > > > > >\n> > > > > > > > > > 1) The source buffer is ready\n> > > > > > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > > > > > your generateJpegExifMetadata()) are available\n> > > > > > > > > >\n> > > > > > > > > > without waiting for the whole request to complete ?\n> > > > > > > > > >\n> > > > > > > > > > In your last patch you introduce support for handling\n> > > > > > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > > > > > happens when the buffer is completed without verifying that the\n> > > > > > > > > > required metadata to populate the EXIF tags are available.\n> > > > > > > > > >\n> > > > > > > > > > Does this match your understanding ?\n> > > > > > > > >\n> > > > > > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > > > > > >\n> > > > > > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > > > > > completed, so perhaps we could expect them to be notified with signal\n> > > > > > > >\n> > > > > > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > > > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > > > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > > > > > support early metadata completion but could signl buffer being\n> > > > > > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > > > > > metadata before buffers and implement compliance tests to validate\n> > > > > > > > that, I don't think we can assume anything about ordering.\n> > > > > > > >\n> > > > > > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > > > > > you wouldn't like it(?\n> > > > > > > > >\n> > > > > > > >\n> > > > > > > > I don't think signalling metadata at requestComplete time -after-\n> > > > > > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > > > > > what they think about this.\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > One possible way to handle this is to\n> > > > > > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > > > > > otherwise skip it\n> > > > > > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > > > > > walk the list of completed buffers to process and run post-processing\n> > > > > > > > >\n> > > > > > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > > > > > general outline ?\n> > > > > > > > > >\n> > > > > > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > > > > > >\n> > > > > > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > > > > > multiple Mapped stream on one Direct.\n> > > > > > > > > >\n> > > > > > > > > > I would take from this series:\n> > > > > > > > > > 2/9\n> > > > > > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > > > > > >\n> > > > > > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > > > > > quite some time breaking this into two though in the beginning haha...\n> > > > > > > > >\n> > > > > > > >\n> > > > > > > > Sorry about that, but the patch is really about supporting multiple\n> > > > > > > > mapped streams, isn't it ?\n> > > > > > >\n> > > > > > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > > > > > supporting multiple mapped streams...\n> > > > > >\n> > > > > > Do you mean 3/9, right ?\n> > > > > >\n> > > > > > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > > > > > about \"supporting multiple mapped streams\"\n> > > > >\n> > > > > Ah okay, thanks :)\n> > > > >\n> > > > > >\n> > > > > > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> > > > > >\n> > > > > > Careful this patch contains a bunch of unrelated refactories at the\n> > > > > > end\n> > > > >\n> > > > > Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> > > > > remove it when uploading :)\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > No worries, I can still squash it.\n> > > > > > >\n> > > > > > > >\n> > > > > > > > > > 5/9\n> > > > > > > > > > 6/9\n> > > > > > > > >\n> > > > > > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > > > > > >\n> > > > > > > >\n> > > > > > > > Thanks\n> > > > > > >\n> > > > > > > We still have some ongoing discussion in the patch. Please help take a\n> > > > > > > look when you have time. Thanks!\n> > > > > >\n> > > > > > On 5/9 and 6/9 ?\n> > > > > >\n> > > > > > 5/9 I sent my R-b tag\n> > > > > > 6/9 I suggested a new commit message but the rest is ok with me\n> > > > >\n> > > > > Yeah on 6/9 I assumed you would have questions about introducing\n> > > > > CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > > > > > >\n> > > > > > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > > > > > >\n> > > > > > > > > Sure :)\n> > > > > > > > >\n> > > > > > > > > > support for handling bufferCompleted() (without handling\n> > > > > > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > > > > > metadataAvailable on top.\n> > > > > > > > >\n> > > > > > > > > I suggest to support signal metadataAvailable first, because:\n> > > > > > > > >\n> > > > > > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > > > > > blocked by post processing, and we could immediately notify the\n> > > > > > > > > application with a partial result.\n> > > > > >\n> > > > > > ack\n> > > > > >\n> > > > > > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > > > > > Android adapter.\n> > > > > > > > > - I've actually done this split in this order, but haven't sent them\n> > > > > >\n> > > > > > fine with this ordering\n> > > > > >\n> > > > > > > > > as patches yet. You can check on gitlab first:\n> > > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > > > > > >\n> > > > > > > >\n> > > > > > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > > > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > > > > > >\n> > > > > > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > > > > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > > > > > that doesn't support it yet.\n> > > > > > >\n> > > > > >\n> > > > > > That's great, thanks\n> > > > > >\n> > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > > > > > others. What do you think ?\n> > > > > > > > >\n> > > > > > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > > > > > metadata patch into the rest two of the upcoming patches\n> > > > > > > >\n> > > > > > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > > > > > might require more than just two patches :)\n> > > > > > >\n> > > > > > > I really hope I can split them into pieces, while I'm running out of\n> > > > > > > ideas... If you can take a brief look and let me know how you expect\n> > > > > > > them to be split, I'd appreciate that a lot!\n> > > > > > >\n> > > > > > > Some random ideas:\n> > > > > > > On `android: Support partial results with metadataAvailable`:\n> > > > > > >\n> > > > > > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > > > > > the only result.\n> > > > > > > - Add `Camera3ResultDescriptor` and\n> > > > > > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > > > > > migrated), while I'm not sure if it makes sense...\n> > > > > > >\n> > > > > >\n> > > > > > see below\n> > > > > >\n> > > > > > > >\n> > > > > > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > > > > > >\n> > > > > > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > > > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > > > > > streams as soon as they're available\" should become quite natural to\n> > > > > > > > be done on top.\n> > > > > > > >\n> > > > > > > > I presume you will need Camera3ResultDescriptor already when\n> > > > > > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > > > > > >\n> > > > > > > Yes, what I meant is that I find it difficult to keep this patch\n> > > > > > > separated from the upcoming patches, unless we implement it on top of\n> > > > > > > them. In this way though, getting the correct result metadata when\n> > > > > >\n> > > > > > Indeed I think it should happen after support for partial metadata and\n> > > > > > bufferCompleted have been developed\n> > > > > >\n> > > > > > > post-processing with jpeg will be unsupported in the middle. Is that\n> > > > > >\n> > > > > > Why do you think so ? As long as process() happens at requestCompleted\n> > > > > > time, all the metadata info you need will be available in\n> > > > > > Request::metadata\n> > > > > >\n> > > > > > > acceptable?\n> > > > > >\n> > > > > > I still fail to see why doing this in lock-step would bring issues,\n> > > > > > but I admit I only glanced through\n> > > > > > 8 files changed, 621 insertions(+), 277 deletions(-)\n> > > > > >\n> > > > > > Could you please tell me in which of the following steps you'll have\n> > > > > > issues:\n> > > > > >\n> > > > > > 1) Add support for metadataAvailable signal\n> > > > > >\n> > > > > > Support the new signal and allow partial metadata completion. The\n> > > > > > HAL can now call process_capture_results multiple times for the\n> > > > > > same request.\n> > > > > >\n> > > > > > Post-processing is not changed as it happens at requestCompleted\n> > > > > > time, when the buffers are available and the required metadata to\n> > > > > > populate EXIF will be available in Request::metadata() as it used\n> > > > > > to be\n> > > > > >\n> > > > > > 2) Add support for bufferCompleted\n> > > > > >\n> > > > > > Support earlier buffer completion. The HAL should already be\n> > > > > > instrumented to send partial results for the Direct buffer that has\n> > > > > > just completed.\n> > > >\n> > > > I assume this includes sending the direct buffers back to the\n> > > > application when receiving signals of bufferCompleted, while delaying\n> > > > post-processing to requestCompleted.\n> > > >\n> > >\n> > > What makes you think we have to delay post-processing to\n> > > requestComplete ?\n> > >\n> > > My first thought was that as soon as a Direct buffer is ready, if\n> > > any stream is Mapped on it, we can schedule post-processing.\n> >\n> > I assumed that's in the third step (patch). I don't know how to start\n> > post-processing without the functionalities to check if the required\n> > metadata to populate the EXIF tags are available (for JPEG).\n> >\n> > >\n> > > > When I tried to implement this, I encountered an issue that I don't\n> > > > know how to fix: When we send a partial result containing a direct\n> > > > buffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\n> > >\n> > > Yeah, you're now giving it back to the Android framework which is\n> > > probably free to re-use it as it likes.\n> > >\n> > > > Mapped stream/buffer depends on the direct buffer in post-processing,\n> > > > the post-processing might fail due to the lack of the source buffer.\n> > > > This basically stops us from sending buffers back to the application\n> > > > earlier than all post-processing done.\n> > >\n> > > If, for some reason I'm now missing, we have to delay post-processing\n> > > to requestCompleted, then Direct buffers on which another Stream is\n> > > Mapped on have to be delayed to when post-processing is done, as we do\n> > > today. Only Direct buffers with no mapped Stream can be signalled\n> > > earlier.\n> > >\n> > > As said, anticipating post-processing to when both the source buffer\n> > > and the required metadata are available might be desirable, but I\n> > > guess I'm missing something here.\n> >\n> > Exactly, checking the required metadata is mandatory to do\n> > post-processing earlier in partial results.\n> > In our previous plan, it happens after the patch to introduce the\n> > bufferCompleted signal.\n>\n> Friendly ping: any update?\n>\n> BR,\n> Harvey\n>\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > > >\n> > > > Therefore, unless this patch only sets buffers' status and releases\n> > > > direct buffers' fences, it's very hard to be standalone.\n> > > >\n> > > > WDYT?\n> > > >\n> > > > BR,\n> > > > Harvey\n> > > >\n> > > >\n> > > > >\n> > > > > Yes, if we keep the processing (or at least the jpeg one) in\n> > > > > requestComplete, then we won't drop the feature.\n> > > > >\n> > > > > >\n> > > > > > 3) Schedule post-processing as soon as the source buffer is available\n> > > > > >\n> > > > > > Track if for each request the required metadata to populate the EXIF\n> > > > > > tags are available (for JPEG) and schedule post-processing as soon\n> > > > > > as the source buffer (and metadata for JPEG) are available.\n> > > > > >\n> > > > > > This indeed might require multiple patches to implement tracking of\n> > > > > > the metadata and buffer status\n> > > > > >\n> > > > > > All steps should be validated with a pipeline that support\n> > > > > > metadataAvailable and with a pipeline that doesn't.\n> > > > > >\n> > > > > > As far as I can all of this currently happens in a single patch (9/9\n> > > > > > and partially in 8/9). I might be surely missing details and\n> > > > > > implementation issues, but logically the above seems to me a\n> > > > > > reasonable break-down of 8/9 and 9/9 ?\n> > > > >\n> > > > > Yes, I'll try to implement this. Thanks!\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > BR,\n> > > > > > > Harvey\n> > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > BR,\n> > > > > > > > > Harvey\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > WDYT?\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > I might have missed why this is related :)\n> > > > > > > > > > >\n> > > > > > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > > > > > might not have been completed.\n> > > > > > > > > > >\n> > > > > > > > > > > BR,\n> > > > > > > > > > > Harvey\n> > > > > > > > > > >\n> > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > Thanks\n> > > > > > > > > > > > > > > j\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > > > > > -\n> > > > > > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > --\n> > > > > > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E076FBDB1C\n\tfor ;\n\tThu, 2 Jan 2025 15:26:27 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 172F5684CF;\n\tThu, 2 Jan 2025 16:26:27 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9824D684CC\n\tfor ;\n\tThu, 2 Jan 2025 16:26:23 +0100 (CET)","from ideasonboard.com (mob-5-90-142-196.net.vodafone.it\n\t[5.90.142.196])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 142E122F;\n\tThu, 2 Jan 2025 16:25:33 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"a8zrZ7NM\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1735831534;\n\tbh=KPvXNMUKtQtZdqRjaT/PxcDKdx9jB0XhowQq60lZRWw=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=a8zrZ7NM42nOyMitc6GuV9Zn6RiIbwuRr4wdM65bVi916z3QYEqqsygjY9HjSRUfU\n\t3eAYnPSJBOz4r7Pi9PwYlip7aYVLIjQk9jo5Ce4d4u1sl7jt3um58OmNK4aOy2co3H\n\txoBTh9SZ2hi0+wikWA3WnNcVnNl9fRiSdOeIiPow=","Date":"Thu, 2 Jan 2025 16:26:19 +0100","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Jacopo Mondi , \n\tlibcamera-devel@lists.libcamera.org,\n\tHan-Lin Chen ","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","Message-ID":"","References":"\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t\n\t\n\t<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>\n\t\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32917,"web_url":"https://patchwork.libcamera.org/comment/32917/","msgid":"","date":"2025-01-02T15:55:48","subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","submitter":{"id":117,"url":"https://patchwork.libcamera.org/api/people/117/","name":"Cheng-Hao Yang","email":"chenghaoyang@chromium.org"},"content":"Hi Jacopo,\n\nOn Thu, Jan 2, 2025 at 4:26 PM Jacopo Mondi\n wrote:\n>\n> Harvey\n>\n> to me it's clear this thread has reached a length which makes it\n> rather hard to continue the discussion.\n>\n> As I recall the status was that you have broken out\n> [PATCH v4 0/7] Refactor Android HAL before supporting partial result\n>\n> from\n> [PATCH v2 0/8] Signal metadataAvailable and Android partial result\n\nCorrect, and it's close to being fully reviewed.\n\n>\n> and we agreed we should have found a better solution for partial\n> result support in libcamera before handling it in Android.\n\nCorrect as well.\n\n>\n> For this purpose I sent on the 6th December\n> [PATCH 0/8] libcamera: Support partial metadata completion\n> https://patchwork.libcamera.org/project/libcamera/list/?series=4858\n> on which I have cc-ed you and Han-lin\n>\n> The series adds support for partial results to libcamera, and the\n> android handling should be designed on top of this (or any other\n> solution that we decide is the correct one).\n>\n> I haven't received feedback on that series, have you perhaps missed it ?\n\nYes, sorry, we missed that. Thanks for the heads-up.\nI'll start reviewing the series.\n\nBR,\nHarvey\n\n>\n>\n>\n> On Thu, Jan 02, 2025 at 02:11:57PM +0100, Cheng-Hao Yang wrote:\n> > Hi Jacopo,\n> >\n> > On Fri, Dec 13, 2024 at 9:59 AM Cheng-Hao Yang\n> > wrote:\n> > >\n> > > Hi Jacopo,\n> > >\n> > > On Fri, Dec 13, 2024 at 4:26 PM Jacopo Mondi\n> > > wrote:\n> > > >\n> > > > Hi Harvey\n> > > >\n> > > > On Thu, Dec 12, 2024 at 05:00:23PM +0800, Cheng-Hao Yang wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On Wed, Dec 4, 2024 at 11:35 PM Cheng-Hao Yang\n> > > > > wrote:\n> > > > > >\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On Wed, Dec 4, 2024 at 11:16 PM Jacopo Mondi\n> > > > > > wrote:\n> > > > > > >\n> > > > > > > Hi Harvey\n> > > > > > >\n> > > > > > > On Wed, Dec 04, 2024 at 10:31:16PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On Wed, Dec 4, 2024 at 6:48 PM Jacopo Mondi\n> > > > > > > > wrote:\n> > > > > > > > >\n> > > > > > > > > Hi Harvey\n> > > > > > > > >\n> > > > > > > > > On Wed, Dec 04, 2024 at 06:29:53PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > Hi Jacopo,\n> > > > > > > > > >\n> > > > > > > > > > On Wed, Dec 4, 2024 at 5:42 PM Jacopo Mondi\n> > > > > > > > > > wrote:\n> > > > > > > > > > >\n> > > > > > > > > > > Hi Harvey\n> > > > > > > > > > >\n> > > > > > > > > > > On Tue, Dec 03, 2024 at 10:33:22PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > >\n> > > > > > > > > > > > On Tue, Dec 3, 2024 at 1:04 AM Jacopo Mondi\n> > > > > > > > > > > > wrote:\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > On Fri, Nov 29, 2024 at 05:17:30PM +0800, Cheng-Hao Yang wrote:\n> > > > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > On Fri, Nov 29, 2024 at 5:05 PM Cheng-Hao Yang\n> > > > > > > > > > > > > > wrote:\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > Hi Jacopo,\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > On Thu, Nov 28, 2024 at 11:35 PM Jacopo Mondi\n> > > > > > > > > > > > > > > wrote:\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > Hi Harvey\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > On Wed, Nov 27, 2024 at 09:25:58AM +0000, Harvey Yang wrote:\n> > > > > > > > > > > > > > > > > From: Han-Lin Chen \n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > With partial result, some metadata, which needs to be added into Exif,\n> > > > > > > > > > > > > > > > > may be sent back to framework earlier before Jpeg post-processing.\n> > > > > > > > > > > > > > > > > Add a type JpegExifMetadata associated with StreamBuffer to store the values,\n> > > > > > > > > > > > > > > > > so Jpeg post-processing doesn't need to reference to current metadata.\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > Signed-off-by: Han-Lin Chen \n> > > > > > > > > > > > > > > > > Co-developed-by: Harvey Yang \n> > > > > > > > > > > > > > > > > Signed-off-by: Harvey Yang \n> > > > > > > > > > > > > > > > > ---\n> > > > > > > > > > > > > > > > > src/android/camera_device.cpp | 26 ++++++++++++++++++++++++\n> > > > > > > > > > > > > > > > > src/android/camera_device.h | 2 ++\n> > > > > > > > > > > > > > > > > src/android/camera_request.h | 6 ++++++\n> > > > > > > > > > > > > > > > > src/android/camera_stream.h | 4 ++++\n> > > > > > > > > > > > > > > > > src/android/jpeg/post_processor_jpeg.cpp | 12 ++++++-----\n> > > > > > > > > > > > > > > > > 5 files changed, 45 insertions(+), 5 deletions(-)\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > > index 9fd851bc8..e085e18b2 100644\n> > > > > > > > > > > > > > > > > --- a/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > > +++ b/src/android/camera_device.cpp\n> > > > > > > > > > > > > > > > > @@ -1250,6 +1250,10 @@ void CameraDevice::requestComplete(Request *request)\n> > > > > > > > > > > > > > > > > CameraStream *stream = iter->first;\n> > > > > > > > > > > > > > > > > StreamBuffer *buffer = iter->second;\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > + if (stream->isJpegStream()) {\n> > > > > > > > > > > > > > > > > + generateJpegExifMetadata(descriptor, buffer);\n> > > > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > no {} for single line statements\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Done\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > FrameBuffer *src = request->findBuffer(stream->stream());\n> > > > > > > > > > > > > > > > > if (!src) {\n> > > > > > > > > > > > > > > > > LOG(HAL, Error) << \"Failed to find a source stream buffer\";\n> > > > > > > > > > > > > > > > > @@ -1443,6 +1447,28 @@ void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> > > > > > > > > > > > > > > > > callbacks_->notify(callbacks_, ¬ify);\n> > > > > > > > > > > > > > > > > }\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > +/*\n> > > > > > > > > > > > > > > > > + * Set jpeg metadata used to generate EXIF in the JPEG post processing.\n> > > > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > > > +void CameraDevice::generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > > > + StreamBuffer *buffer) const\n> > > > > > > > > > > > > > > > > +{\n> > > > > > > > > > > > > > > > > + const ControlList &metadata = request->request_->metadata();\n> > > > > > > > > > > > > > > > > + auto &jpegExifMetadata = buffer->jpegExifMetadata;\n> > > > > > > > > > > > > > > > > + jpegExifMetadata.emplace(StreamBuffer::JpegExifMetadata());\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > > + const int64_t exposureTime = metadata.get(controls::ExposureTime).value_or(0);\n> > > > > > > > > > > > > > > > > + jpegExifMetadata->sensorExposureTime = exposureTime;\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > > + /*\n> > > > > > > > > > > > > > > > > + * todo: Android Sensitivity should only include analog gain X digital\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > \\todo\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > done\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > + * gain from sensor. Digital gain on ISP shouldn't be included.\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > mmm, I guess how the gain is split between analogue and digital on the\n> > > > > > > > > > > > > > > > sensor is up to the IPA implementation, and currently I only see vc4\n> > > > > > > > > > > > > > > > handling it and it sets it on the ISP.\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > I wonder if you couldn't simply use AnalogueGain here\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > I think the comment here is assuming the further changes that use\n> > > > > > > > > > > > > > AnalogueGain directly here, while might not be needed in this patch...\n> > > > > > > > > > > > > > Removed.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I'm not sure I get this comment in full. I was suggesting to use\n> > > > > > > > > > > > > AnalogueGain directly here, but if it isn't required, I'm fine with\n> > > > > > > > > > > > > keeping a \\todo\n> > > > > > > > > > > >\n> > > > > > > > > > > > Yeah I know, while I think using AnalogueGain or even a \\todo is\n> > > > > > > > > > > > irrelevant in this patch. We may add a separate one to use\n> > > > > > > > > > > > AnalogueGain directly :)\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > I'll remember (hopefully) when we use AnalogueGain here in the\n> > > > > > > > > > > > > > following patches.\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > + * Calculate sensitivity accordingly when we can differentiate\n> > > > > > > > > > > > > > > > > + * the source of digital gains.\n> > > > > > > > > > > > > > > > > + */\n> > > > > > > > > > > > > > > > > + jpegExifMetadata->sensorSensitivityISO = 100;\n> > > > > > > > > > > > > > > > > +}\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > > /*\n> > > > > > > > > > > > > > > > > * Produce a set of fixed result metadata.\n> > > > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > > > diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> > > > > > > > > > > > > > > > > index 815a695d1..3c46ff918 100644\n> > > > > > > > > > > > > > > > > --- a/src/android/camera_device.h\n> > > > > > > > > > > > > > > > > +++ b/src/android/camera_device.h\n> > > > > > > > > > > > > > > > > @@ -102,6 +102,8 @@ private:\n> > > > > > > > > > > > > > > > > void sendCaptureResult(Camera3RequestDescriptor *request) const;\n> > > > > > > > > > > > > > > > > void setBufferStatus(StreamBuffer &buffer,\n> > > > > > > > > > > > > > > > > StreamBuffer::Status status);\n> > > > > > > > > > > > > > > > > + void generateJpegExifMetadata(Camera3RequestDescriptor *request,\n> > > > > > > > > > > > > > > > > + StreamBuffer *buffer) const;\n> > > > > > > > > > > > > > > > > std::unique_ptr getResultMetadata(\n> > > > > > > > > > > > > > > > > const Camera3RequestDescriptor &descriptor) const;\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > diff --git a/src/android/camera_request.h b/src/android/camera_request.h\n> > > > > > > > > > > > > > > > > index bd75d4595..bd87b36fd 100644\n> > > > > > > > > > > > > > > > > --- a/src/android/camera_request.h\n> > > > > > > > > > > > > > > > > +++ b/src/android/camera_request.h\n> > > > > > > > > > > > > > > > > @@ -44,6 +44,11 @@ public:\n> > > > > > > > > > > > > > > > > StreamBuffer(StreamBuffer &&);\n> > > > > > > > > > > > > > > > > StreamBuffer &operator=(StreamBuffer &&);\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > + struct JpegExifMetadata {\n> > > > > > > > > > > > > > > > > + int64_t sensorExposureTime;\n> > > > > > > > > > > > > > > > > + int32_t sensorSensitivityISO;\n> > > > > > > > > > > > > > > > > + };\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > > > CameraStream *stream;\n> > > > > > > > > > > > > > > > > buffer_handle_t *camera3Buffer;\n> > > > > > > > > > > > > > > > > std::unique_ptr frameBuffer;\n> > > > > > > > > > > > > > > > > @@ -51,6 +56,7 @@ public:\n> > > > > > > > > > > > > > > > > Status status = Status::Success;\n> > > > > > > > > > > > > > > > > const libcamera::FrameBuffer *srcBuffer = nullptr;\n> > > > > > > > > > > > > > > > > std::unique_ptr dstBuffer;\n> > > > > > > > > > > > > > > > > + std::optional jpegExifMetadata;\n> > > > > > > > > > > > > > > > > Camera3RequestDescriptor *request;\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > private:\n> > > > > > > > > > > > > > > > > diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > > index 30f64f690..47cd7ab85 100644\n> > > > > > > > > > > > > > > > > --- a/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > > +++ b/src/android/camera_stream.h\n> > > > > > > > > > > > > > > > > @@ -125,6 +125,10 @@ public:\n> > > > > > > > > > > > > > > > > const libcamera::StreamConfiguration &configuration() const;\n> > > > > > > > > > > > > > > > > libcamera::Stream *stream() const;\n> > > > > > > > > > > > > > > > > CameraStream *sourceStream() const { return sourceStream_; }\n> > > > > > > > > > > > > > > > > + bool isJpegStream() const\n> > > > > > > > > > > > > > > > > + {\n> > > > > > > > > > > > > > > > > + return camera3Stream_->format == HAL_PIXEL_FORMAT_BLOB;\n> > > > > > > > > > > > > > > > > + }\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > int configure();\n> > > > > > > > > > > > > > > > > int process(StreamBuffer *streamBuffer);\n> > > > > > > > > > > > > > > > > diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > > index f5a90785d..48782b574 100644\n> > > > > > > > > > > > > > > > > --- a/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > > +++ b/src/android/jpeg/post_processor_jpeg.cpp\n> > > > > > > > > > > > > > > > > @@ -112,8 +112,11 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > const FrameBuffer &source = *streamBuffer->srcBuffer;\n> > > > > > > > > > > > > > > > > CameraBuffer *destination = streamBuffer->dstBuffer.get();\n> > > > > > > > > > > > > > > > > + const std::optional &jpegExifMetadata =\n> > > > > > > > > > > > > > > > > + streamBuffer->jpegExifMetadata;\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > ASSERT(destination->numPlanes() == 1);\n> > > > > > > > > > > > > > > > > + ASSERT(jpegExifMetadata.has_value());\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > This means it's not optional, isn't it ?\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > Means it's not std::nullopt. Any suggestions?\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > yeah, what I meant was \"if you ASSERT()\" you expect it to be always\n> > > > > > > > > > > > > populated, so std::optional<> doesn't bring any value. But you\n> > > > > > > > > > > > > probably want to make sure that jpegExifMetadata has been populated\n> > > > > > > > > > > > > when this function is called, so feel free to keep std::optional<>\n> > > > > > > > > > > >\n> > > > > > > > > > > > Sure :)\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > BR,\n> > > > > > > > > > > > > > Harvey\n> > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > const CameraMetadata &requestMetadata = streamBuffer->request->settings_;\n> > > > > > > > > > > > > > > > > CameraMetadata *resultMetadata = streamBuffer->request->resultMetadata_.get();\n> > > > > > > > > > > > > > > > > @@ -139,15 +142,14 @@ void PostProcessorJpeg::process(StreamBuffer *streamBuffer)\n> > > > > > > > > > > > > > > > > */\n> > > > > > > > > > > > > > > > > exif.setTimestamp(std::time(nullptr), 0ms);\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_EXPOSURE_TIME, &entry);\n> > > > > > > > > > > > > > > > > - exif.setExposureTime(ret ? *entry.data.i64 : 0);\n> > > > > > > > > > > > > > > > > + /* Exif requires nsec for exposure time */\n> > > > > > > > > > > > > > > > > + exif.setExposureTime(jpegExifMetadata->sensorExposureTime * 1000);\n> > > > > > > > > > > > > > > > > + exif.setISO(jpegExifMetadata->sensorSensitivityISO);\n> > > > > > > > > > > > > > > > > +\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > StreamBuffer has a pointer to the Camera3RequestDescriptor it belongs\n> > > > > > > > > > > > > > > > to. From there you could get the Request metadata as you currently do\n> > > > > > > > > > > > > > > > in CameraDevice::generateJpegExifMetadata().\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > That's a very good question, only that I wonder how we can handle the\n> > > > > > > > > > > > > > > threading issue properly.\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > The current implementation of `Request::metadata()` [1] doesn't seem\n> > > > > > > > > > > > > > > to consider race conditions, and our goal is to support partial results,\n> > > > > > > > > > > > > > > which means that the post processor thread might try to access metadata\n> > > > > > > > > > > > > > > when the request is still being processed in the pipeline handler, which\n> > > > > > > > > > > > > > > might set further metadata tags.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Is this an issue about threading or about the order in which metadata\n> > > > > > > > > > > > > and buffer completes ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > What I meant is the threading issue, while the order is also an issue, true.\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I see in the next patch you call generateJpegExifMetadata() on a\n> > > > > > > > > > > > > ((Mapped|Internal) && Jpeg) bufferComplete event.\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1227\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > And immediately after, if such a buffer has been completed, you call\n> > > > > > > > > > > > > process() on it\n> > > > > > > > > > > > > https://gitlab.freedesktop.org/camera/libcamera/-/commit/b731fe488badef2861da914913290e16afb716c8#88faf21e943da09c94a5b31cd420d91c35371290_1193_1251\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > Is this right ?\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > So when the buffer completes, you inspect the so-far-completed\n> > > > > > > > > > > > > metadata and extract ExposureTime (and AnalogueGain eventually).\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > 1) Why you don't do that at metadataAvailable time ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > Makes a lot of sense. Will try to apply in the next patch.\n> > > > > > > > > > > >\n> > > > > > > > > > > > > 2) What does guarantee that the pipeline has populated ExposureTime at\n> > > > > > > > > > > > > the time the buffer to process has completed ?\n> > > > > > > > > > > >\n> > > > > > > > > > > > That's a very good question. I'd like your input: Do you think we\n> > > > > > > > > > > > should pause the jpeg process until all necessary metadata tags are\n> > > > > > > > > > > > available?\n> > > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > Good question..\n> > > > > > > > > > >\n> > > > > > > > > > > As I see in PostProcessorJpeg::process(StreamBuffer *streamBuffer) the\n> > > > > > > > > > > jpegExifMetadata metadata are used to populate the EXIF data. In the\n> > > > > > > > > > > same function the jpeg post-processor is run, so at the moment EXIF\n> > > > > > > > > > > metadata creation and post-processing happens at the same time and is in\n> > > > > > > > > > > facts run at requestComplete() time.\n> > > > > > > > > > >\n> > > > > > > > > > > Let me get the requirements straight here: I presume the idea is to\n> > > > > > > > > > > produce the jpeg frame as soon as:\n> > > > > > > > > > >\n> > > > > > > > > > > 1) The source buffer is ready\n> > > > > > > > > > > 2) All the required metadata to populate the EXIF tags (collected by\n> > > > > > > > > > > your generateJpegExifMetadata()) are available\n> > > > > > > > > > >\n> > > > > > > > > > > without waiting for the whole request to complete ?\n> > > > > > > > > > >\n> > > > > > > > > > > In your last patch you introduce support for handling\n> > > > > > > > > > > Camera::bufferCompleted and the newly introduced\n> > > > > > > > > > > Camera::metadataAvailable signals but the post-processing still\n> > > > > > > > > > > happens when the buffer is completed without verifying that the\n> > > > > > > > > > > required metadata to populate the EXIF tags are available.\n> > > > > > > > > > >\n> > > > > > > > > > > Does this match your understanding ?\n> > > > > > > > > >\n> > > > > > > > > > Yes, it's my understanding as well. Thanks for sorting the logic out.\n> > > > > > > > > >\n> > > > > > > > > > Han-lin proposes a question though: The ExposureTime (and the upcoming\n> > > > > > > > > > AnalogueGain) metadata should already be ready when the jpeg buffer is\n> > > > > > > > > > completed, so perhaps we could expect them to be notified with signal\n> > > > > > > > >\n> > > > > > > > > Yes, \"should\", however there's nothing that forces a pipeline handler\n> > > > > > > > > to guarantee that they are. In example, a pipeline handler (especially\n> > > > > > > > > an existing one not ported to use the metadataAvailable signal) might not\n> > > > > > > > > support early metadata completion but could signl buffer being\n> > > > > > > > > completed. Unless we make it mandatory for pipelines to signal those\n> > > > > > > > > metadata before buffers and implement compliance tests to validate\n> > > > > > > > > that, I don't think we can assume anything about ordering.\n> > > > > > > > >\n> > > > > > > > > > metadataAvailable earlier than bufferCompleted being called? He\n> > > > > > > > > > suggests adding a WARNING/ERROR log and a \\todo, although I feel that\n> > > > > > > > > > you wouldn't like it(?\n> > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > I don't think signalling metadata at requestComplete time -after-\n> > > > > > > > > bufferCompleted is a warning/error condition. I'll check with other\n> > > > > > > > > what they think about this.\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > One possible way to handle this is to\n> > > > > > > > > > > 1) at bufferCompleted time accumulate a list of completed FrameBuffers\n> > > > > > > > > > > to process (and signal them with process_capture_request). Check if the\n> > > > > > > > > > > Request::metadata() where all the partial metadata are accumulated\n> > > > > > > > > > > contains the necessary tags for EXIF, if they do call process()\n> > > > > > > > > > > otherwise skip it\n> > > > > > > > > > > 2) at metadataAvailable time check if the EXIF are there, if they are\n> > > > > > > > > > > walk the list of completed buffers to process and run post-processing\n> > > > > > > > > >\n> > > > > > > > > > Yes, basically this is what I have in my mind as well, thanks.\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > I'm sure there will be challanges, but what do you think of this as a\n> > > > > > > > > > > general outline ?\n> > > > > > > > > > >\n> > > > > > > > > > > Now, how to get there... I really think your last patch should be\n> > > > > > > > > > > broken out to pieces. It's too much stuff to digest in one go.\n> > > > > > > > > > >\n> > > > > > > > > > > There are patches in this series that can be fast-tracked,\n> > > > > > > > > > > specifically the ones that make it possible to correctly handle\n> > > > > > > > > > > multiple Mapped stream on one Direct.\n> > > > > > > > > > >\n> > > > > > > > > > > I would take from this series:\n> > > > > > > > > > > 2/9\n> > > > > > > > > > > 3/9 + 4/9 squased together as suggested in the review\n> > > > > > > > > >\n> > > > > > > > > > Sure, I can squash them if you think that makes more sense. (I spent\n> > > > > > > > > > quite some time breaking this into two though in the beginning haha...\n> > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > Sorry about that, but the patch is really about supporting multiple\n> > > > > > > > > mapped streams, isn't it ?\n> > > > > > > >\n> > > > > > > > Yeah, and I still think that the 3rd patch doesn't contribute to\n> > > > > > > > supporting multiple mapped streams...\n> > > > > > >\n> > > > > > > Do you mean 3/9, right ?\n> > > > > > >\n> > > > > > > Feel free to keep it separate if you prefer, as long as 4/9 is clearly\n> > > > > > > about \"supporting multiple mapped streams\"\n> > > > > >\n> > > > > > Ah okay, thanks :)\n> > > > > >\n> > > > > > >\n> > > > > > > > Haven't uploaded as a patch, but you can take a look at the new commit message:\n> > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/0c5b916a26878776d29982f8ab43c1155daa1f07\n> > > > > > >\n> > > > > > > Careful this patch contains a bunch of unrelated refactories at the\n> > > > > > > end\n> > > > > >\n> > > > > > Yeah, my linter on nvim insists on cleaning up the whole file. I'll\n> > > > > > remove it when uploading :)\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > No worries, I can still squash it.\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > > 5/9\n> > > > > > > > > > > 6/9\n> > > > > > > > > >\n> > > > > > > > > > I think I'll split this into two: one to use CAMERA3_MSG_ERROR_RESULT,\n> > > > > > > > > > and the other one to report CAMERA3_MSG_ERROR_REQUEST out of order.\n> > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > Thanks\n> > > > > > > >\n> > > > > > > > We still have some ongoing discussion in the patch. Please help take a\n> > > > > > > > look when you have time. Thanks!\n> > > > > > >\n> > > > > > > On 5/9 and 6/9 ?\n> > > > > > >\n> > > > > > > 5/9 I sent my R-b tag\n> > > > > > > 6/9 I suggested a new commit message but the rest is ok with me\n> > > > > >\n> > > > > > Yeah on 6/9 I assumed you would have questions about introducing\n> > > > > > CAMERA3_MSG_ERROR_RESULT. If not, I'm happy to keep it as is.\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > > 7/9 (pending review, I asked Laurent to give there a look)\n> > > > > > > > > > >\n> > > > > > > > > > > Then, prepare a series to fast-track JPEG processing and support\n> > > > > > > > > > > partial results. I would defer 1/9 to that series. Then introduce\n> > > > > > > > > >\n> > > > > > > > > > Sure :)\n> > > > > > > > > >\n> > > > > > > > > > > support for handling bufferCompleted() (without handling\n> > > > > > > > > > > metadataAvailable yet, this will validate that the HAL works with\n> > > > > > > > > > > platforms that do not deliver early metadata). Then handle\n> > > > > > > > > > > metadataAvailable on top.\n> > > > > > > > > >\n> > > > > > > > > > I suggest to support signal metadataAvailable first, because:\n> > > > > > > > > >\n> > > > > > > > > > - It actually involves less changes, as a set of metadata wouldn't be\n> > > > > > > > > > blocked by post processing, and we could immediately notify the\n> > > > > > > > > > application with a partial result.\n> > > > > > >\n> > > > > > > ack\n> > > > > > >\n> > > > > > > > > > - metadataAvailable and bufferCompleted are actually both new to\n> > > > > > > > > > Android adapter.\n> > > > > > > > > > - I've actually done this split in this order, but haven't sent them\n> > > > > > >\n> > > > > > > fine with this ordering\n> > > > > > >\n> > > > > > > > > > as patches yet. You can check on gitlab first:\n> > > > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/c25f483b7c4b2e6b2d1fc2eb5cf2851db874ab11\n> > > > > > > > > > https://gitlab.freedesktop.org/chenghaoyang/libcamera/-/commit/97b8c426656755b4a6e21cc8d8397ccd92395481\n> > > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > Sure, as long as we test with CTS with both pipelines supporting\n> > > > > > > > > metadataAvailable and pipelines not supporting metadataAvailable\n> > > > > > > >\n> > > > > > > > Before the whole series get merged, I'll test it on mtkisp7, which\n> > > > > > > > will support the new signal. We also have ipu3 on soraka-libcamera\n> > > > > > > > that doesn't support it yet.\n> > > > > > > >\n> > > > > > >\n> > > > > > > That's great, thanks\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > Again, there will certainly be challanges, and I'm not even sure the\n> > > > > > > > > > > breakdown of the second part is possible, but indeed I would separate\n> > > > > > > > > > > 1/9 and 8/9 9/9 from the rest of the patches and fast-track the\n> > > > > > > > > > > others. What do you think ?\n> > > > > > > > > >\n> > > > > > > > > > Sure, just one question: Do you think we should merge this Jpeg\n> > > > > > > > > > metadata patch into the rest two of the upcoming patches\n> > > > > > > > >\n> > > > > > > > > Please note supporting metadataAvailable and bufferComplete in the HAL\n> > > > > > > > > might require more than just two patches :)\n> > > > > > > >\n> > > > > > > > I really hope I can split them into pieces, while I'm running out of\n> > > > > > > > ideas... If you can take a brief look and let me know how you expect\n> > > > > > > > them to be split, I'd appreciate that a lot!\n> > > > > > > >\n> > > > > > > > Some random ideas:\n> > > > > > > > On `android: Support partial results with metadataAvailable`:\n> > > > > > > >\n> > > > > > > > - Add a patch that only adds `kMaxMetadataPackIndex`, and use it in\n> > > > > > > > the only result.\n> > > > > > > > - Add `Camera3ResultDescriptor` and\n> > > > > > > > `Camera3RequestDescriptor::finalResult_` (with resultMetadata_\n> > > > > > > > migrated), while I'm not sure if it makes sense...\n> > > > > > > >\n> > > > > > >\n> > > > > > > see below\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > > (metadataAvailable & bufferCompleted)? It's indeed a bit difficult to\n> > > > > > > > > > implement it without Camera3ResultDescriptor and instances available.\n> > > > > > > > >\n> > > > > > > > > Ideally, after having plumbed in support for metadataAvailable and\n> > > > > > > > > bufferComplete we should be in a state where \"processing Mapped\n> > > > > > > > > streams as soon as they're available\" should become quite natural to\n> > > > > > > > > be done on top.\n> > > > > > > > >\n> > > > > > > > > I presume you will need Camera3ResultDescriptor already when\n> > > > > > > > > handling (metadataAvailable & bufferCompleted) or am I wrong ?\n> > > > > > > >\n> > > > > > > > Yes, what I meant is that I find it difficult to keep this patch\n> > > > > > > > separated from the upcoming patches, unless we implement it on top of\n> > > > > > > > them. In this way though, getting the correct result metadata when\n> > > > > > >\n> > > > > > > Indeed I think it should happen after support for partial metadata and\n> > > > > > > bufferCompleted have been developed\n> > > > > > >\n> > > > > > > > post-processing with jpeg will be unsupported in the middle. Is that\n> > > > > > >\n> > > > > > > Why do you think so ? As long as process() happens at requestCompleted\n> > > > > > > time, all the metadata info you need will be available in\n> > > > > > > Request::metadata\n> > > > > > >\n> > > > > > > > acceptable?\n> > > > > > >\n> > > > > > > I still fail to see why doing this in lock-step would bring issues,\n> > > > > > > but I admit I only glanced through\n> > > > > > > 8 files changed, 621 insertions(+), 277 deletions(-)\n> > > > > > >\n> > > > > > > Could you please tell me in which of the following steps you'll have\n> > > > > > > issues:\n> > > > > > >\n> > > > > > > 1) Add support for metadataAvailable signal\n> > > > > > >\n> > > > > > > Support the new signal and allow partial metadata completion. The\n> > > > > > > HAL can now call process_capture_results multiple times for the\n> > > > > > > same request.\n> > > > > > >\n> > > > > > > Post-processing is not changed as it happens at requestCompleted\n> > > > > > > time, when the buffers are available and the required metadata to\n> > > > > > > populate EXIF will be available in Request::metadata() as it used\n> > > > > > > to be\n> > > > > > >\n> > > > > > > 2) Add support for bufferCompleted\n> > > > > > >\n> > > > > > > Support earlier buffer completion. The HAL should already be\n> > > > > > > instrumented to send partial results for the Direct buffer that has\n> > > > > > > just completed.\n> > > > >\n> > > > > I assume this includes sending the direct buffers back to the\n> > > > > application when receiving signals of bufferCompleted, while delaying\n> > > > > post-processing to requestCompleted.\n> > > > >\n> > > >\n> > > > What makes you think we have to delay post-processing to\n> > > > requestComplete ?\n> > > >\n> > > > My first thought was that as soon as a Direct buffer is ready, if\n> > > > any stream is Mapped on it, we can schedule post-processing.\n> > >\n> > > I assumed that's in the third step (patch). I don't know how to start\n> > > post-processing without the functionalities to check if the required\n> > > metadata to populate the EXIF tags are available (for JPEG).\n> > >\n> > > >\n> > > > > When I tried to implement this, I encountered an issue that I don't\n> > > > > know how to fix: When we send a partial result containing a direct\n> > > > > buffer, the buffer's lifetime cannot be guaranteed. Therefore, if a\n> > > >\n> > > > Yeah, you're now giving it back to the Android framework which is\n> > > > probably free to re-use it as it likes.\n> > > >\n> > > > > Mapped stream/buffer depends on the direct buffer in post-processing,\n> > > > > the post-processing might fail due to the lack of the source buffer.\n> > > > > This basically stops us from sending buffers back to the application\n> > > > > earlier than all post-processing done.\n> > > >\n> > > > If, for some reason I'm now missing, we have to delay post-processing\n> > > > to requestCompleted, then Direct buffers on which another Stream is\n> > > > Mapped on have to be delayed to when post-processing is done, as we do\n> > > > today. Only Direct buffers with no mapped Stream can be signalled\n> > > > earlier.\n> > > >\n> > > > As said, anticipating post-processing to when both the source buffer\n> > > > and the required metadata are available might be desirable, but I\n> > > > guess I'm missing something here.\n> > >\n> > > Exactly, checking the required metadata is mandatory to do\n> > > post-processing earlier in partial results.\n> > > In our previous plan, it happens after the patch to introduce the\n> > > bufferCompleted signal.\n> >\n> > Friendly ping: any update?\n> >\n> > BR,\n> > Harvey\n> >\n> > >\n> > > BR,\n> > > Harvey\n> > >\n> > > >\n> > > > >\n> > > > > Therefore, unless this patch only sets buffers' status and releases\n> > > > > direct buffers' fences, it's very hard to be standalone.\n> > > > >\n> > > > > WDYT?\n> > > > >\n> > > > > BR,\n> > > > > Harvey\n> > > > >\n> > > > >\n> > > > > >\n> > > > > > Yes, if we keep the processing (or at least the jpeg one) in\n> > > > > > requestComplete, then we won't drop the feature.\n> > > > > >\n> > > > > > >\n> > > > > > > 3) Schedule post-processing as soon as the source buffer is available\n> > > > > > >\n> > > > > > > Track if for each request the required metadata to populate the EXIF\n> > > > > > > tags are available (for JPEG) and schedule post-processing as soon\n> > > > > > > as the source buffer (and metadata for JPEG) are available.\n> > > > > > >\n> > > > > > > This indeed might require multiple patches to implement tracking of\n> > > > > > > the metadata and buffer status\n> > > > > > >\n> > > > > > > All steps should be validated with a pipeline that support\n> > > > > > > metadataAvailable and with a pipeline that doesn't.\n> > > > > > >\n> > > > > > > As far as I can all of this currently happens in a single patch (9/9\n> > > > > > > and partially in 8/9). I might be surely missing details and\n> > > > > > > implementation issues, but logically the above seems to me a\n> > > > > > > reasonable break-down of 8/9 and 9/9 ?\n> > > > > >\n> > > > > > Yes, I'll try to implement this. Thanks!\n> > > > > >\n> > > > > > BR,\n> > > > > > Harvey\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > BR,\n> > > > > > > > Harvey\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > >\n> > > > > > > > > > BR,\n> > > > > > > > > > Harvey\n> > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > WDYT?\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > [1]: https://git.libcamera.org/libcamera/libcamera.git/tree/src/libcamera/request.cpp#n530\n> > > > > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > I might have missed why this is related :)\n> > > > > > > > > > > >\n> > > > > > > > > > > > Okay, I think I misunderstood the \\todo, but the point is that it\n> > > > > > > > > > > > doesn't handle the threading issue. We should avoid reading it in the\n> > > > > > > > > > > > post processor thread, especially with partial results, as the Request\n> > > > > > > > > > > > might not have been completed.\n> > > > > > > > > > > >\n> > > > > > > > > > > > BR,\n> > > > > > > > > > > > Harvey\n> > > > > > > > > > > >\n> > > > > > > > > > > > >\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > What is the advantage of caching the jpegExifMetadata at\n> > > > > > > > > > > > > > > > CameraDevice::requestComplete() time ?\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > Thanks\n> > > > > > > > > > > > > > > > j\n> > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry);\n> > > > > > > > > > > > > > > > > if (ret)\n> > > > > > > > > > > > > > > > > exif.setAperture(*entry.data.f);\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > - ret = resultMetadata->getEntry(ANDROID_SENSOR_SENSITIVITY, &entry);\n> > > > > > > > > > > > > > > > > - exif.setISO(ret ? *entry.data.i32 : 100);\n> > > > > > > > > > > > > > > > > -\n> > > > > > > > > > > > > > > > > exif.setFlash(Exif::Flash::FlashNotPresent);\n> > > > > > > > > > > > > > > > > exif.setWhiteBalance(Exif::WhiteBalance::Auto);\n> > > > > > > > > > > > > > > > >\n> > > > > > > > > > > > > > > > > --\n> > > > > > > > > > > > > > > > > 2.47.0.338.g60cca15819-goog\n> > > > > > > > > > > > > > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 9B7AAC32C2\n\tfor ;\n\tThu, 2 Jan 2025 15:56:02 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 87151684D9;\n\tThu, 2 Jan 2025 16:56:01 +0100 (CET)","from mail-lj1-x22c.google.com (mail-lj1-x22c.google.com\n\t[IPv6:2a00:1450:4864:20::22c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 13666684CC\n\tfor ;\n\tThu, 2 Jan 2025 16:56:00 +0100 (CET)","by mail-lj1-x22c.google.com with SMTP id\n\t38308e7fff4ca-300392cc4caso132855131fa.3\n\tfor ;\n\tThu, 02 Jan 2025 07:55:59 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"e4bgJax4\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1735833359; x=1736438159;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=dzaZLHdu2YO2AeXB/eRKchsJBvAL7fN292ij2R6uaTE=;\n\tb=e4bgJax4MxUvJ0gi+fmJS1k2PAuxpW6RHGjdosieKfPAskfB3nfuDY6LNI8Xswb86t\n\t/+kKJfJ/VHlpUlIg83s5C4mHIqNaw7RpKaIwSV7re6AevO+PfA7Dai4K5NI98jVfaiN0\n\tP3XDtlusMfv9MPS9sWomA/2xvq3zrrEr6UDmc=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1735833359; x=1736438159;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=dzaZLHdu2YO2AeXB/eRKchsJBvAL7fN292ij2R6uaTE=;\n\tb=q7K45GceJRcyh5kXFP2dCPu7N8uPdmQQTBczPsaW4AMH8tAuByIReTEyr+pqLm0xhM\n\tlpPwsGzPjh7wjQlabnUsQalJsAj+LNi8rp7XHPQ963JNau18WtOSq5m2rSTjOSnEaOW6\n\t4D3mpfYdNcufybX76PxQ8EPuW9StfC0xFKYGPDMpnwozlpXAtrXf8OjqFAvLeLY6JZrq\n\tfrnhmFdxlUpqu1LDfmrpV9xC40W9jAANmTakhb89UU2LCuW1FmqLSTteU4c04JLMk0BK\n\tIlKwXniJD3n5XgUjG22t/aCgjWkarfpWKKO0HhNObPeVfDP4pEeM/aEG22M6Erc0hV3+\n\tbVYw==","X-Gm-Message-State":"AOJu0YwCi0jZQvk7nQ9r2jH8DWaYak2Vy+J2pcpObI64g+OhrNSNZrjN\n\t2xHeN/WHx74gBGh4F2U7UmzietOdBdaMO8cEpanPSHjRK8nJheDoCSN2D3nS4GybklERx/SHzTl\n\tMlDyLsjVygVutLMuoOvJh1pDasbfJ12mDUGAw","X-Gm-Gg":"ASbGnctK4tqu6cPiCi78aVTTVNk9wY1IfPDgApuE523GZEXKlGS1xm2RR33aWkm/n7g\n\tDCKyMWHiSigsZj/6DK3y9BFRkzwKRPPDIPVg6","X-Google-Smtp-Source":"AGHT+IE5f7pW+srZtX/vLYfXNg3ac+CXFPmS5o9rNnqQSPEFmnvakZQejG4t0EouCiTmsCRvlra4L7SFKcjejc6CWmI=","X-Received":"by 2002:a05:651c:2225:b0:302:1e65:f2ab with SMTP id\n\t38308e7fff4ca-30468567c26mr157998001fa.20.1735833358929;\n\tThu, 02 Jan 2025 07:55:58 -0800 (PST)","MIME-Version":"1.0","References":"\n\t\n\t\n\t\n\t<4ck4zuyvba4yvoda3wsnsrkxtozffuvcobwetbf3rset2xkdgf@uvuklfrq4bkx>\n\t\n\t\n\t<63ihfj7bp3lffvwwhnkvjsm74dpxjvgsnslamnrkgkgwavo4vv@zey6yypxswef>\n\t\n\t\n\t","In-Reply-To":"","From":"Cheng-Hao Yang ","Date":"Thu, 2 Jan 2025 16:55:48 +0100","Message-ID":"","Subject":"Re: [PATCH v2 8/9] android: Add JpegExifMetadata to store tags\n\tsetting into Exif","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHan-Lin Chen ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]