From patchwork Thu Jan 14 10:40:35 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 10862 X-Patchwork-Delegate: paul.elder@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 78131BD808 for ; Thu, 14 Jan 2021 10:40:59 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4261E680F4; Thu, 14 Jan 2021 11:40:59 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="cgEr7MPP"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6BC41680E6 for ; Thu, 14 Jan 2021 11:40:58 +0100 (CET) Received: from pyrite.rasen.tech (unknown [IPv6:2400:4051:61:600:2c71:1b79:d06d:5032]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 098452B3; Thu, 14 Jan 2021 11:40:56 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1610620858; bh=vS2pGjGLDSCCr5zG6kLgXXEkp62yX9U+4rSq50WDYs8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=cgEr7MPPI984Ga6REmL4n2P691LOOlZKBwxIH3Yoz1mP/x1un7Gf6Vixey/8kJCDL rRR4hg1jT0FtYnkUIg3u/IUEF8gi36zhf9jqpE/uYaIAbmVpWxCUHueswaZbrzj4Xr jekEdVPz19I6X0BN0okB/W+6hVIHHNYg1vHASAJ0= From: Paul Elder To: libcamera-devel@lists.libcamera.org Date: Thu, 14 Jan 2021 19:40:35 +0900 Message-Id: <20210114104035.302968-7-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210114104035.302968-1-paul.elder@ideasonboard.com> References: <20210114104035.302968-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 6/6] android: Set result metadata and EXIF fields based on request metadata X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Set the following android result metadata: - ANDROID_LENS_FOCAL_LENGTH - ANDROID_LENS_APERTURE - ANDROID_JPEG_GPS_TIMESTAMP - ANDROID_JPEG_THUMBNAIL_SIZE - ANDROID_JPEG_THUMBNAIL_QUALITY - ANDROID_JPEG_GPS_COORDINATES - ANDROID_JPEG_GPS_PROCESSING_METHOD - ANDROID_JPEG_SIZE - ANDROID_JPEG_QUALITY - ANDROID_JPEG_ORIENTATION And the following EXIF fields: - GPSDatestamp - GPSTimestamp - GPSLocation - GPSLatitudeRef - GPSLatitude - GPSLongitudeRef - GPSLongitude - GPSAltitudeRef - GPSAltitude - GPSProcessingMethod - FocalLength - ExposureTime - FNumber - ISO - Flash - WhiteBalance - SubsecTime - SubsecTimeOriginal - SubsecTimeDigitized Based on android request metadata. This allows the following CTS tests to pass: - android.hardware.camera2.cts.StillCaptureTest#testFocalLengths - android.hardware.camera2.cts.StillCaptureTest#testJpegExif Signed-off-by: Paul Elder --- src/android/camera_device.cpp | 27 +++++- src/android/camera_device.h | 5 +- src/android/camera_stream.cpp | 7 +- src/android/camera_stream.h | 4 +- src/android/jpeg/post_processor_jpeg.cpp | 116 +++++++++++++++++++---- src/android/jpeg/post_processor_jpeg.h | 5 +- src/android/jpeg/thumbnailer.h | 1 + src/android/post_processor.h | 3 +- 8 files changed, 136 insertions(+), 32 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index ed47c7cd..8d697080 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -296,8 +296,9 @@ MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer, */ CameraDevice::Camera3RequestDescriptor::Camera3RequestDescriptor( - Camera *camera, unsigned int frameNumber, unsigned int numBuffers) - : frameNumber_(frameNumber), numBuffers_(numBuffers) + Camera *camera, unsigned int frameNumber, unsigned int numBuffers, + CameraMetadata &settings) + : frameNumber_(frameNumber), numBuffers_(numBuffers), settings_(settings) { buffers_ = new camera3_stream_buffer_t[numBuffers]; @@ -1700,9 +1701,12 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques * The descriptor and the associated memory reserved here are freed * at request complete time. */ + /* \todo Handle null request settings */ + CameraMetadata requestMetadata(camera3Request->settings); Camera3RequestDescriptor *descriptor = new Camera3RequestDescriptor(camera_.get(), camera3Request->frame_number, - camera3Request->num_output_buffers); + camera3Request->num_output_buffers, + requestMetadata); LOG(HAL, Debug) << "Queueing Request to libcamera with " << descriptor->numBuffers_ << " HAL streams"; @@ -1815,6 +1819,9 @@ void CameraDevice::requestComplete(Request *request) CameraStream *cameraStream = static_cast(descriptor->buffers_[i].stream->priv); + float focal_length = 1.0; + resultMetadata->addEntry(ANDROID_LENS_FOCAL_LENGTH, &focal_length, 1); + if (cameraStream->camera3Stream().format != HAL_PIXEL_FORMAT_BLOB) continue; @@ -1837,6 +1844,7 @@ void CameraDevice::requestComplete(Request *request) } int ret = cameraStream->process(*buffer, &mapped, + descriptor->settings_, resultMetadata.get()); if (ret) { status = CAMERA3_BUFFER_STATUS_ERROR; @@ -1933,14 +1941,23 @@ CameraDevice::getResultMetadata(Camera3RequestDescriptor *descriptor, /* * \todo Keep this in sync with the actual number of entries. - * Currently: 33 entries, 75 bytes + * Currently: 41 entries, 187 bytes * * Reserve more space for the JPEG metadata set by the post-processor. * Currently: ANDROID_JPEG_SIZE (int32_t), ANDROID_JPEG_QUALITY (byte), * ANDROID_JPEG_ORIENTATION (int32_t) = 3 entries, 9 bytes. + * ANDROID_JPEG_THUMBNAIL_SIZE (int32 x 2) = 8 bytes + * ANDROID_JPEG_THUMBNAIL_QUALITY (int32 x 2) = 8 bytes + * ANDROID_LENS_FOCAL_LENGTH (float) = 4 bytes + * ANDROID_JPEG_GPS_COORDINATES (double x 3) = 24 bytes + * ANDROID_JPEG_GPS_PROCESSING_METHOD (byte x 32) = 32 bytes + * ANDROID_JPEG_GPS_TIMESTAMP (int64) = 8 bytes + * ANDROID_LENS_APERTURE (float) = 4 bytes + * + * add coordinates again? */ std::unique_ptr resultMetadata = - std::make_unique(33, 75); + std::make_unique(41, 187); if (!resultMetadata->isValid()) { LOG(HAL, Error) << "Failed to allocate static metadata"; return nullptr; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index a285d0a1..111a7d8f 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -24,6 +24,7 @@ #include "libcamera/internal/log.h" #include "libcamera/internal/message.h" +#include "camera_metadata.h" #include "camera_stream.h" #include "camera_worker.h" #include "jpeg/encoder.h" @@ -78,7 +79,8 @@ private: struct Camera3RequestDescriptor { Camera3RequestDescriptor(libcamera::Camera *camera, unsigned int frameNumber, - unsigned int numBuffers); + unsigned int numBuffers, + CameraMetadata &settings); ~Camera3RequestDescriptor(); uint32_t frameNumber_; @@ -86,6 +88,7 @@ private: camera3_stream_buffer_t *buffers_; std::vector> frameBuffers_; std::unique_ptr request_; + CameraMetadata settings_; }; struct Camera3StreamConfiguration { diff --git a/src/android/camera_stream.cpp b/src/android/camera_stream.cpp index dba351a4..4c8020e5 100644 --- a/src/android/camera_stream.cpp +++ b/src/android/camera_stream.cpp @@ -96,12 +96,15 @@ int CameraStream::configure() } int CameraStream::process(const libcamera::FrameBuffer &source, - MappedCamera3Buffer *dest, CameraMetadata *metadata) + MappedCamera3Buffer *dest, + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata) { if (!postProcessor_) return 0; - return postProcessor_->process(source, dest->maps()[0], metadata); + return postProcessor_->process(source, dest->maps()[0], + requestMetadata, resultMetadata); } FrameBuffer *CameraStream::getBuffer() diff --git a/src/android/camera_stream.h b/src/android/camera_stream.h index cc9d5470..298ffbf6 100644 --- a/src/android/camera_stream.h +++ b/src/android/camera_stream.h @@ -119,7 +119,9 @@ public: int configure(); int process(const libcamera::FrameBuffer &source, - MappedCamera3Buffer *dest, CameraMetadata *metadata); + MappedCamera3Buffer *dest, + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata); libcamera::FrameBuffer *getBuffer(); void putBuffer(libcamera::FrameBuffer *buffer); diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp index 436a50f8..13ad3777 100644 --- a/src/android/jpeg/post_processor_jpeg.cpp +++ b/src/android/jpeg/post_processor_jpeg.cpp @@ -25,6 +25,21 @@ PostProcessorJpeg::PostProcessorJpeg(CameraDevice *const device) { } +int PostProcessorJpeg::configureThumbnailer(const Size &size, + const PixelFormat &pixelFormat) +{ + thumbnailer_.configure(size, pixelFormat); + StreamConfiguration thCfg; + thCfg.size = thumbnailer_.size(); + thCfg.pixelFormat = pixelFormat; + if (thumbnailEncoder_.configure(thCfg) != 0) { + LOG(JPEG, Error) << "Failed to configure thumbnail encoder"; + return -EINVAL; + } + + return 0; +} + int PostProcessorJpeg::configure(const StreamConfiguration &inCfg, const StreamConfiguration &outCfg) { @@ -40,16 +55,11 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg, streamSize_ = outCfg.size; - thumbnailer_.configure(inCfg.size, inCfg.pixelFormat); - StreamConfiguration thCfg = inCfg; - thCfg.size = thumbnailer_.size(); - if (thumbnailEncoder_.configure(thCfg) != 0) { - LOG(JPEG, Error) << "Failed to configure thumbnail encoder"; - return -EINVAL; - } + int ret = configureThumbnailer(inCfg.size, inCfg.pixelFormat); + if (ret) + return ret; encoder_ = std::make_unique(); - return encoder_->configure(inCfg); } @@ -80,17 +90,22 @@ void PostProcessorJpeg::generateThumbnail(const FrameBuffer &source, int PostProcessorJpeg::process(const FrameBuffer &source, Span destination, - CameraMetadata *metadata) + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata) { if (!encoder_) return 0; + camera_metadata_ro_entry_t entry; + int ret; + /* Set EXIF metadata for various tags. */ Exif exif; - /* \todo Set Make and Model from external vendor tags. */ - exif.setMake("libcamera"); - exif.setModel("cameraModel"); - exif.setOrientation(cameraDevice_->orientation()); + exif.setMake(cameraDevice_->cameraMake()); + exif.setModel(cameraDevice_->cameraModel()); + + ret = requestMetadata.getEntry(ANDROID_JPEG_ORIENTATION, &entry); + exif.setOrientation(ret ? cameraDevice_->orientation() : *entry.data.i32); exif.setSize(streamSize_); /* * We set the frame's EXIF timestamp as the time of encode. @@ -99,11 +114,68 @@ int PostProcessorJpeg::process(const FrameBuffer &source, */ exif.setTimestamp(std::time(nullptr)); - std::vector thumbnail; - generateThumbnail(source, &thumbnail); - if (!thumbnail.empty()) + exif.setExposureTime(0); + ret = requestMetadata.getEntry(ANDROID_LENS_APERTURE, &entry); + if (!ret) { + exif.setAperture(*entry.data.f); + resultMetadata->addEntry(ANDROID_LENS_APERTURE, entry.data.f, 1); + } + exif.setISO(100); + exif.setFlash(Exif::Flash::FlashNotPresent); + exif.setWhiteBalance(Exif::WhiteBalance::Auto); + + exif.setSubsecTime(0); + exif.setSubsecTimeOriginal(0); + exif.setSubsecTimeDigitized(0); + + float focal_length = 1.0; + exif.setFocalLength(focal_length); + + ret = requestMetadata.getEntry(ANDROID_JPEG_GPS_TIMESTAMP, &entry); + if (!ret) { + exif.setGPSDateTimestamp(*entry.data.i64); + resultMetadata->addEntry(ANDROID_JPEG_GPS_TIMESTAMP, + entry.data.i64, 1); + } + + ret = requestMetadata.getEntry(ANDROID_JPEG_THUMBNAIL_SIZE, &entry); + if (!ret) { + /* \todo Handle non-matching aspect ratios */ + /* \todo Handle non-zero orientations */ + const int32_t *data = entry.data.i32; + Size thumbnailSize = { static_cast(data[0]), + static_cast(data[1]) }; + + std::vector thumbnail; + if (thumbnailSize != Size(0, 0)) { + configureThumbnailer(thumbnailSize, thumbnailer_.pixelFormat()); + generateThumbnail(source, &thumbnail); + } exif.setThumbnail(thumbnail, Exif::Compression::JPEG); + resultMetadata->addEntry(ANDROID_JPEG_THUMBNAIL_SIZE, data, 2); + + /* \todo Use this quality as a parameter to the encoder */ + ret = requestMetadata.getEntry(ANDROID_JPEG_THUMBNAIL_QUALITY, &entry); + if (!ret) + resultMetadata->addEntry(ANDROID_JPEG_THUMBNAIL_QUALITY, entry.data.u8, 1); + } + + ret = requestMetadata.getEntry(ANDROID_JPEG_GPS_COORDINATES, &entry); + if (!ret) { + exif.setGPSLocation(entry.data.d); + resultMetadata->addEntry(ANDROID_JPEG_GPS_COORDINATES, + entry.data.d, 3); + } + + ret = requestMetadata.getEntry(ANDROID_JPEG_GPS_PROCESSING_METHOD, &entry); + if (!ret) { + std::string method(entry.data.u8, entry.data.u8 + 32); + exif.setGPSMethod(method); + resultMetadata->addEntry(ANDROID_JPEG_GPS_PROCESSING_METHOD, + entry.data.u8, 32); + } + if (exif.generate() != 0) LOG(JPEG, Error) << "Failed to generate valid EXIF data"; @@ -133,13 +205,15 @@ int PostProcessorJpeg::process(const FrameBuffer &source, blob->jpeg_size = jpeg_size; /* Update the JPEG result Metadata. */ - metadata->addEntry(ANDROID_JPEG_SIZE, &jpeg_size, 1); + resultMetadata->addEntry(ANDROID_JPEG_SIZE, &jpeg_size, 1); - const uint32_t jpeg_quality = 95; - metadata->addEntry(ANDROID_JPEG_QUALITY, &jpeg_quality, 1); + ret = requestMetadata.getEntry(ANDROID_JPEG_QUALITY, &entry); + const uint32_t jpeg_quality = ret ? 95 : *entry.data.u8; + resultMetadata->addEntry(ANDROID_JPEG_QUALITY, &jpeg_quality, 1); - const uint32_t jpeg_orientation = 0; - metadata->addEntry(ANDROID_JPEG_ORIENTATION, &jpeg_orientation, 1); + ret = requestMetadata.getEntry(ANDROID_JPEG_ORIENTATION, &entry); + const uint32_t jpeg_orientation = ret ? 0 : *entry.data.i32; + resultMetadata->addEntry(ANDROID_JPEG_ORIENTATION, &jpeg_orientation, 1); return 0; } diff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h index 5afa831c..d07c9fe5 100644 --- a/src/android/jpeg/post_processor_jpeg.h +++ b/src/android/jpeg/post_processor_jpeg.h @@ -26,11 +26,14 @@ public: const libcamera::StreamConfiguration &outcfg) override; int process(const libcamera::FrameBuffer &source, libcamera::Span destination, - CameraMetadata *metadata) override; + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata) override; private: void generateThumbnail(const libcamera::FrameBuffer &source, std::vector *thumbnail); + int configureThumbnailer(const libcamera::Size &size, + const libcamera::PixelFormat &pixelFormat); CameraDevice *const cameraDevice_; std::unique_ptr encoder_; diff --git a/src/android/jpeg/thumbnailer.h b/src/android/jpeg/thumbnailer.h index 98f11833..f393db47 100644 --- a/src/android/jpeg/thumbnailer.h +++ b/src/android/jpeg/thumbnailer.h @@ -22,6 +22,7 @@ public: void createThumbnail(const libcamera::FrameBuffer &source, std::vector *dest); const libcamera::Size &size() const { return targetSize_; } + const libcamera::PixelFormat &pixelFormat() const { return pixelFormat_; } private: libcamera::Size computeThumbnailSize() const; diff --git a/src/android/post_processor.h b/src/android/post_processor.h index e0f91880..bda93bb4 100644 --- a/src/android/post_processor.h +++ b/src/android/post_processor.h @@ -22,7 +22,8 @@ public: const libcamera::StreamConfiguration &outCfg) = 0; virtual int process(const libcamera::FrameBuffer &source, libcamera::Span destination, - CameraMetadata *metadata) = 0; + const CameraMetadata &requestMetadata, + CameraMetadata *resultMetadata) = 0; }; #endif /* __ANDROID_POST_PROCESSOR_H__ */