From patchwork Mon Oct 26 14:01:34 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 10255 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 2EDC5BDB13 for ; Mon, 26 Oct 2020 14:02:42 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id EE77D61E91; Mon, 26 Oct 2020 15:02:41 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=uajain.com header.i=@uajain.com header.b="L2jLFVYD"; dkim-atps=neutral Received: from mail.uajain.com (static.126.159.217.95.clients.your-server.de [95.217.159.126]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id B263461E71 for ; Mon, 26 Oct 2020 15:02:40 +0100 (CET) From: Umang Jain DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=uajain.com; s=mail; t=1603720960; bh=QmYKKfebxTENrzsoOI3L5vBR4EfpOxRpBW+hX3xGy6g=; h=From:To:Cc:Subject:In-Reply-To:References; b=L2jLFVYDY92+INuUlS+Ga0IkXrxSBuGWy/eAGCGslLtUT0Qvxbz91zDhuiElUXwec uCGASryRghRFkjtQVsfvU71T1ObsAbAe8p4NMBWOjdQksSoKnKSKY45hb8wow/Klnv Flrk3G5M1fLtCcGtahugYKau3qdFjiZB9x5Hi1st+adcJIdYftQj/DuBGZS+3xauT5 He4dxCfg6NuufLsSomYdayHEV7sEbg/2D8KgcSXB8v4qEVlYedcq0++3MBTxIIfdFk P2x9MQ7LAQvPwJYBZDFrRWUPzJB/+trJ9MZB6z2/1XLXbe7+iri1npX1uGuyTHeL5q 2hRcoNU/tBVqw== To: libcamera-devel@lists.libcamera.org Date: Mon, 26 Oct 2020 19:31:34 +0530 Message-Id: <20201026140134.44166-4-email@uajain.com> In-Reply-To: <20201026140134.44166-1-email@uajain.com> References: <20201026140134.44166-1-email@uajain.com> Mime-Version: 1.0 Subject: [libcamera-devel] [PATCH 3/3] android: jpeg: exif: Embed a JPEG-encoded thumbnail X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add a basic image thumbnailer for NV12 frames being captured. It shall generate a thumbnail image to be embedded as a part of EXIF metadata of the frame. The output of the thumbnail will still be NV12. Signed-off-by: Umang Jain --- src/android/jpeg/exif.cpp | 16 +++- src/android/jpeg/exif.h | 1 + src/android/jpeg/post_processor_jpeg.cpp | 35 +++++++- src/android/jpeg/post_processor_jpeg.h | 8 +- src/android/jpeg/thumbnailer.cpp | 109 +++++++++++++++++++++++ src/android/jpeg/thumbnailer.h | 37 ++++++++ src/android/meson.build | 1 + 7 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 src/android/jpeg/thumbnailer.cpp create mode 100644 src/android/jpeg/thumbnailer.h diff --git a/src/android/jpeg/exif.cpp b/src/android/jpeg/exif.cpp index d21534a..24197bd 100644 --- a/src/android/jpeg/exif.cpp +++ b/src/android/jpeg/exif.cpp @@ -75,8 +75,16 @@ Exif::~Exif() if (exifData_) free(exifData_); - if (data_) + if (data_) { + /* + * Reset thumbnail data to avoid getting double-freed by + * libexif. It is owned by the caller (i.e. PostProcessorJpeg). + */ + data_->data = nullptr; + data_->size = 0; + exif_data_unref(data_); + } if (mem_) exif_mem_unref(mem_); @@ -268,6 +276,12 @@ void Exif::setOrientation(int orientation) setShort(EXIF_IFD_0, EXIF_TAG_ORIENTATION, value); } +void Exif::setThumbnail(std::vector &thumbnail) +{ + data_->data = thumbnail.data(); + data_->size = thumbnail.size(); +} + [[nodiscard]] int Exif::generate() { if (exifData_) { diff --git a/src/android/jpeg/exif.h b/src/android/jpeg/exif.h index 12c27b6..bd54a31 100644 --- a/src/android/jpeg/exif.h +++ b/src/android/jpeg/exif.h @@ -26,6 +26,7 @@ public: void setOrientation(int orientation); void setSize(const libcamera::Size &size); + void setThumbnail(std::vector &thumbnail); void setTimestamp(time_t timestamp); libcamera::Span data() const { return { exifData_, size_ }; } diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp index c56f1b2..416e831 100644 --- a/src/android/jpeg/post_processor_jpeg.cpp +++ b/src/android/jpeg/post_processor_jpeg.cpp @@ -9,7 +9,6 @@ #include "../camera_device.h" #include "../camera_metadata.h" -#include "encoder_libjpeg.h" #include "exif.h" #include @@ -39,11 +38,42 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg, } streamSize_ = outCfg.size; + + thumbnailer_.configure(inCfg.size, inCfg.pixelFormat); + StreamConfiguration thCfg = inCfg; + thCfg.size = thumbnailer_.size(); + if (thumbnailEncoder_.configure(thCfg) != 0) { + LOG(JPEG, Error) << "Failed to configure thumbnail encoder"; + return -EINVAL; + } + encoder_ = std::make_unique(); return encoder_->configure(inCfg); } +void PostProcessorJpeg::generateThumbnail(const FrameBuffer &source, + std::vector &thumbnail) +{ + /* Stores the raw scaled-down thumbnail bytes. */ + std::vector rawThumbnail; + + thumbnailer_.scaleBuffer(source, rawThumbnail); + + if (rawThumbnail.data()) { + thumbnail.reserve(rawThumbnail.capacity()); + + int jpeg_size = thumbnailEncoder_.encode( + { rawThumbnail.data(), rawThumbnail.capacity() }, + { thumbnail.data(), thumbnail.capacity() }, + { }); + thumbnail.resize(jpeg_size); + + LOG(JPEG, Info) << "Thumbnail compress returned " + << jpeg_size << " bytes"; + } +} + int PostProcessorJpeg::process(const FrameBuffer &source, Span destination, CameraMetadata *metadata) @@ -64,6 +94,9 @@ int PostProcessorJpeg::process(const FrameBuffer &source, * second, it is good enough. */ exif.setTimestamp(std::time(nullptr)); + std::vector thumbnail; + generateThumbnail(source, thumbnail); + exif.setThumbnail(thumbnail); if (exif.generate() != 0) LOG(JPEG, Error) << "Failed to generate valid EXIF data"; diff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h index 3706cec..3894231 100644 --- a/src/android/jpeg/post_processor_jpeg.h +++ b/src/android/jpeg/post_processor_jpeg.h @@ -8,12 +8,13 @@ #define __ANDROID_POST_PROCESSOR_JPEG_H__ #include "../post_processor.h" +#include "encoder_libjpeg.h" +#include "thumbnailer.h" #include #include "libcamera/internal/buffer.h" -class Encoder; class CameraDevice; class PostProcessorJpeg : public PostProcessor @@ -28,9 +29,14 @@ public: CameraMetadata *metadata) override; private: + void generateThumbnail(const libcamera::FrameBuffer &source, + std::vector &thumbnail); + CameraDevice *const cameraDevice_; std::unique_ptr encoder_; libcamera::Size streamSize_; + EncoderLibJpeg thumbnailEncoder_; + Thumbnailer thumbnailer_; }; #endif /* __ANDROID_POST_PROCESSOR_JPEG_H__ */ diff --git a/src/android/jpeg/thumbnailer.cpp b/src/android/jpeg/thumbnailer.cpp new file mode 100644 index 0000000..f880ffb --- /dev/null +++ b/src/android/jpeg/thumbnailer.cpp @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * thumbnailer.cpp - Simple image thumbnailer + */ + +#include "thumbnailer.h" + +#include + +#include "libcamera/internal/file.h" +#include "libcamera/internal/log.h" + +using namespace libcamera; + +LOG_DEFINE_CATEGORY(Thumbnailer) + +Thumbnailer::Thumbnailer() : valid_(false) +{ +} + +void Thumbnailer::configure(const Size &sourceSize, PixelFormat pixelFormat) +{ + sourceSize_ = sourceSize; + pixelFormat_ = pixelFormat; + + if (pixelFormat_ != formats::NV12) { + LOG (Thumbnailer, Error) << "Failed to configure: Pixel Format " + << pixelFormat_.toString() << " unsupported."; + return; + } + + targetSize_ = computeThumbnailSize(); + + valid_ = true; +} + +/* + * The Exif specification recommends the width of the thumbnail to be a + * mutiple of 16 (section 4.8.1). Hence, compute the corresponding height + * keeping the aspect ratio same as of the source. + */ +Size Thumbnailer::computeThumbnailSize() +{ + unsigned int targetHeight; + unsigned int targetWidth = 160; + + targetHeight = targetWidth * sourceSize_.height / sourceSize_.width; + + if (targetHeight & 1) + targetHeight++; + + return Size(targetWidth, targetHeight); +} + +void +Thumbnailer::scaleBuffer(const FrameBuffer &source, + std::vector &destination) +{ + MappedFrameBuffer frame(&source, PROT_READ); + if (!frame.isValid()) { + LOG(Thumbnailer, Error) << "Failed to map FrameBuffer : " + << strerror(frame.error()); + return; + } + + if (!valid_) { + LOG(Thumbnailer, Error) << "config is unconfigured or invalid."; + return; + } + + const unsigned int sw = sourceSize_.width; + const unsigned int sh = sourceSize_.height; + const unsigned int tw = targetSize_.width; + const unsigned int th = targetSize_.height; + + /* Image scaling block implementing nearest-neighbour algorithm. */ + unsigned char *src = static_cast(frame.maps()[0].data()); + unsigned char *src_c = src + sh * sw; + unsigned char *src_cb, *src_cr; + unsigned char *dst_y, *src_y; + + size_t dstSize = (th * tw) + ((th / 2) * tw); + destination.reserve(dstSize); + unsigned char *dst = destination.data(); + unsigned char *dst_c = dst + th * tw; + + for (unsigned int y = 0; y < th; y += 2) { + unsigned int sourceY = (sh * y + th / 2) / th; + + dst_y = dst + y * tw; + src_y = src + sw * sourceY; + src_cb = src_c + (sourceY / 2) * sw + 0; + src_cr = src_c + (sourceY / 2) * sw + 1; + + for (unsigned int x = 0; x < tw; x += 2) { + unsigned int sourceX = (sw * x + tw / 2) / tw; + + dst_y[x] = src_y[sourceX]; + dst_y[tw + x] = src_y[sw + sourceX]; + dst_y[x + 1] = src_y[sourceX + 1]; + dst_y[tw + x + 1] = src_y[sw + sourceX + 1]; + + dst_c[(y / 2) * tw + x + 0] = src_cb[(sourceX / 2) * 2]; + dst_c[(y / 2) * tw + x + 1] = src_cr[(sourceX / 2) * 2]; + } + } + } diff --git a/src/android/jpeg/thumbnailer.h b/src/android/jpeg/thumbnailer.h new file mode 100644 index 0000000..b769619 --- /dev/null +++ b/src/android/jpeg/thumbnailer.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * thumbnailer.h - Simple image thumbnailer + */ +#ifndef __ANDROID_JPEG_THUMBNAILER_H__ +#define __ANDROID_JPEG_THUMBNAILER_H__ + +#include + +#include "libcamera/internal/buffer.h" +#include "libcamera/internal/formats.h" + +class Thumbnailer +{ +public: + Thumbnailer(); + + void configure(const libcamera::Size &sourceSize, + libcamera::PixelFormat pixelFormat); + void scaleBuffer(const libcamera::FrameBuffer &source, + std::vector &dest); + libcamera::Size size() const { return targetSize_; } + +private: + libcamera::Size computeThumbnailSize(); + + libcamera::PixelFormat pixelFormat_; + libcamera::Size sourceSize_; + libcamera::Size targetSize_; + + bool valid_; + +}; + +#endif /* __ANDROID_JPEG_THUMBNAILER_H__ */ diff --git a/src/android/meson.build b/src/android/meson.build index f72376a..3d4d3be 100644 --- a/src/android/meson.build +++ b/src/android/meson.build @@ -25,6 +25,7 @@ android_hal_sources = files([ 'jpeg/encoder_libjpeg.cpp', 'jpeg/exif.cpp', 'jpeg/post_processor_jpeg.cpp', + 'jpeg/thumbnailer.cpp', ]) android_camera_metadata_sources = files([