From patchwork Wed Oct 21 08:08:05 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 10158 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 12DD1C3D3C for ; Wed, 21 Oct 2020 08:08:20 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id D3A32610B6; Wed, 21 Oct 2020 10:08:19 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=uajain.com header.i=@uajain.com header.b="taApur+1"; dkim-atps=neutral Received: from mail.uajain.com (static.126.159.217.95.clients.your-server.de [95.217.159.126]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1DF2A610B6 for ; Wed, 21 Oct 2020 10:08:18 +0200 (CEST) From: Umang Jain DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=uajain.com; s=mail; t=1603267697; bh=X7zxhOa5JXahgqrLkw2Bdm3RCeCrIrqS6RA00dk0QZI=; h=From:To:Cc:Subject:In-Reply-To:References; b=taApur+1qN+Bk4F31gIk3NiIkoJQeYAKIkR308z3yByEO3t1pdTAFuLrHYWKVKyoc rlYmcERr8Riz5DH2+IWC8SnWnoFE/xQk6HxEWlVuG5cKVDWYAhzTGWSzuiQW4Ouas8 ug+L6QgABF6E5/w4hPp5r2Du0Rx2vYqKlt4UMxhb6GcAD6daUaa8gaHsCgZCukfoid 1sxf3q6L3nJSy6EkWM2opGyA3XdzXxwehIoxSPC848r1pa+KFP7hEaPo2l3HpRqpZI dhLeXn1Of3UG3Nmu0LbEiAcYUa9E7g9QY19A65fgO8xT7caN6O7bPVdLhSrkRVsVTQ KZYcujSAkAj+A== To: libcamera-devel@lists.libcamera.org Date: Wed, 21 Oct 2020 13:38:05 +0530 Message-Id: <20201021080806.46636-2-email@uajain.com> In-Reply-To: <20201021080806.46636-1-email@uajain.com> References: <20201021080806.46636-1-email@uajain.com> Mime-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH 1/2] android: jpeg: encoder_libjpeg Allow encoding raw frame bytes X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Allow encoding frame which are directly handed over to the encoder via a span or vector i.e. a raw frame bytes. Introduce an overloaded EncoderLibJpeg::encode() with libcamera::Span source parameter to achieve this functionality. This makes the libjpeg-encoder a bit flexible for use case such as compressing a thumbnail generated for Exif. Signed-off-by: Umang Jain Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/android/jpeg/encoder_libjpeg.cpp | 37 +++++++++++++++++----------- src/android/jpeg/encoder_libjpeg.h | 7 ++++-- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/android/jpeg/encoder_libjpeg.cpp b/src/android/jpeg/encoder_libjpeg.cpp index f11e004..bfaf9d5 100644 --- a/src/android/jpeg/encoder_libjpeg.cpp +++ b/src/android/jpeg/encoder_libjpeg.cpp @@ -104,9 +104,9 @@ int EncoderLibJpeg::configure(const StreamConfiguration &cfg) return 0; } -void EncoderLibJpeg::compressRGB(const libcamera::MappedBuffer *frame) +void EncoderLibJpeg::compressRGB(const libcamera::Span &frame) { - unsigned char *src = static_cast(frame->maps()[0].data()); + unsigned char *src = static_cast(frame.data()); /* \todo Stride information should come from buffer configuration. */ unsigned int stride = pixelFormatInfo_->stride(compress_.image_width, 0); @@ -122,7 +122,7 @@ void EncoderLibJpeg::compressRGB(const libcamera::MappedBuffer *frame) * Compress the incoming buffer from a supported NV format. * This naively unpacks the semi-planar NV12 to a YUV888 format for libjpeg. */ -void EncoderLibJpeg::compressNV(const libcamera::MappedBuffer *frame) +void EncoderLibJpeg::compressNV(const libcamera::Span &frame) { uint8_t tmprowbuf[compress_.image_width * 3]; @@ -144,7 +144,7 @@ void EncoderLibJpeg::compressNV(const libcamera::MappedBuffer *frame) unsigned int cb_pos = nvSwap_ ? 1 : 0; unsigned int cr_pos = nvSwap_ ? 0 : 1; - const unsigned char *src = static_cast(frame->maps()[0].data()); + const unsigned char *src = static_cast(frame.data()); const unsigned char *src_c = src + y_stride * compress_.image_height; JSAMPROW row_pointer[1]; @@ -179,17 +179,10 @@ void EncoderLibJpeg::compressNV(const libcamera::MappedBuffer *frame) } } -int EncoderLibJpeg::encode(const FrameBuffer *source, +int EncoderLibJpeg::encode(const libcamera::Span &frame, const libcamera::Span &dest, const libcamera::Span &exifData) { - MappedFrameBuffer frame(source, PROT_READ); - if (!frame.isValid()) { - LOG(JPEG, Error) << "Failed to map FrameBuffer : " - << strerror(frame.error()); - return frame.error(); - } - unsigned char *destination = dest.data(); unsigned long size = dest.size(); @@ -215,11 +208,27 @@ int EncoderLibJpeg::encode(const FrameBuffer *source, << "x" << compress_.image_height; if (nv_) - compressNV(&frame); + compressNV(frame); else - compressRGB(&frame); + compressRGB(frame); jpeg_finish_compress(&compress_); return size; } + +int EncoderLibJpeg::encode(const FrameBuffer *source, + const libcamera::Span &dest, + const libcamera::Span &exifData) +{ + MappedFrameBuffer frame(source, PROT_READ); + if (!frame.isValid()) { + LOG(JPEG, Error) << "Failed to map FrameBuffer : " + << strerror(frame.error()); + return frame.error(); + } + + libcamera::Span src = frame.maps()[0]; + + return encode(src, dest, exifData); +} diff --git a/src/android/jpeg/encoder_libjpeg.h b/src/android/jpeg/encoder_libjpeg.h index 934caef..bf2e512 100644 --- a/src/android/jpeg/encoder_libjpeg.h +++ b/src/android/jpeg/encoder_libjpeg.h @@ -21,13 +21,16 @@ public: ~EncoderLibJpeg(); int configure(const libcamera::StreamConfiguration &cfg) override; + int encode(const libcamera::Span &frames, + const libcamera::Span &destination, + const libcamera::Span &exifData); int encode(const libcamera::FrameBuffer *source, const libcamera::Span &destination, const libcamera::Span &exifData) override; private: - void compressRGB(const libcamera::MappedBuffer *frame); - void compressNV(const libcamera::MappedBuffer *frame); + void compressRGB(const libcamera::Span &frame); + void compressNV(const libcamera::Span &frame); struct jpeg_compress_struct compress_; struct jpeg_error_mgr jerr_; From patchwork Wed Oct 21 08:08:06 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 10159 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 6998EC3D3C for ; Wed, 21 Oct 2020 08:08:22 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3728561DCA; Wed, 21 Oct 2020 10:08:22 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=uajain.com header.i=@uajain.com header.b="giMKzV8u"; dkim-atps=neutral Received: from mail.uajain.com (static.126.159.217.95.clients.your-server.de [95.217.159.126]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id AEDDA60361 for ; Wed, 21 Oct 2020 10:08:20 +0200 (CEST) From: Umang Jain DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=uajain.com; s=mail; t=1603267700; bh=M0rdj81Gq/K9cAEN3s3eNXgUN4lZ8Uuw8RxwvtSJ9H4=; h=From:To:Cc:Subject:In-Reply-To:References; b=giMKzV8u0X+I9TYHzkdRM5WyEjlF2DiA1axEwykhOxdHJEZ5LTm5ln1ZzO3k5gmiL nExOge8s3X+5fD/F33g0BhEf6l7PdKt4xScuuPrbf92JhGRYyj/hKOuxAKErMWdDjM GoOb3D4+t03yX8z1sEiFQeWZBccioQYHVu5FVuOjtx0SKTleUiQgaPn748D9ULKODk VzFfhO++aWM3r6Yu69Jy0LAoZHMIXIv6IdBcnERpZjtwbOW0cJm5lZn66osgBrgbqA m4v564xoSqqRPTtv4mCz7yjeBazSshD028oQjIlPT2fJQR3ZBNdwAl7BhB6cTZ3sEN a85tvKslClCLw== To: libcamera-devel@lists.libcamera.org Date: Wed, 21 Oct 2020 13:38:06 +0530 Message-Id: <20201021080806.46636-3-email@uajain.com> In-Reply-To: <20201021080806.46636-1-email@uajain.com> References: <20201021080806.46636-1-email@uajain.com> Mime-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH 2/2] android: jpeg: Add a basic NV12 image thumbnailer X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add a basic image thumbnailer for NV12 frames being captured. It shall generate a thumbnail image to be embedded as a part of EXIF metadata of the frame. The output of the thumbnail will still be NV12. Signed-off-by: Umang Jain --- src/android/jpeg/post_processor_jpeg.cpp | 39 +++++++++ src/android/jpeg/post_processor_jpeg.h | 3 + src/android/jpeg/thumbnailer.cpp | 100 +++++++++++++++++++++++ src/android/jpeg/thumbnailer.h | 40 +++++++++ src/android/meson.build | 1 + 5 files changed, 183 insertions(+) create mode 100644 src/android/jpeg/thumbnailer.cpp create mode 100644 src/android/jpeg/thumbnailer.h diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp index 9d452b7..f5f1f78 100644 --- a/src/android/jpeg/post_processor_jpeg.cpp +++ b/src/android/jpeg/post_processor_jpeg.cpp @@ -11,6 +11,7 @@ #include "../camera_metadata.h" #include "encoder_libjpeg.h" #include "exif.h" +#include "thumbnailer.h" #include @@ -44,6 +45,32 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg, return encoder_->configure(inCfg); } +void PostProcessorJpeg::generateThumbnail(const libcamera::Span &source, + std::vector &thumbnail) +{ + libcamera::Span destination; + Thumbnailer thumbnailer; + + thumbnailer.configure(streamSize_, formats::NV12); + libcamera::Size targetSize = thumbnailer.computeThumbnailSize(); + thumbnailer.scaleBuffer(source, thumbnail); + + if (thumbnail.data()) { + StreamConfiguration thumbnailCfg; + + std::unique_ptr encoder = + std::make_unique(); + + thumbnailCfg.pixelFormat = formats::NV12; + thumbnailCfg.size = targetSize; + encoder->configure(thumbnailCfg); + int jpeg_size = encoder->encode({ thumbnail.data(), thumbnail.capacity() }, + destination, { }); + LOG(JPEG, Info) << "Thumbnail compress returned " + << jpeg_size << " bytes"; + } +} + int PostProcessorJpeg::process(const libcamera::FrameBuffer *source, const libcamera::Span &destination, CameraMetadata *metadata) @@ -73,6 +100,18 @@ int PostProcessorJpeg::process(const libcamera::FrameBuffer *source, return jpeg_size; } + std::vector thumbnail; + generateThumbnail(destination, thumbnail); + /* + * \todo: Write the compressed thumbnail to a file for inspection. + * (I) Check if we can still write the thumbnail to EXIF here. + * If not, we might need to move the thumbnailer logic to encoder. + * And if we do that, first we need to make sure we get can + * compressed data written to destination first before calling + * jpeg_finish_compress operation somehow. Thumbnailing will + * only occur if we have compressed data available first. + */ + /* * Fill in the JPEG blob header. * diff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h index 62c8650..05601ee 100644 --- a/src/android/jpeg/post_processor_jpeg.h +++ b/src/android/jpeg/post_processor_jpeg.h @@ -28,6 +28,9 @@ public: CameraMetadata *metadata) override; private: + void generateThumbnail(const libcamera::Span &source, + std::vector &thumbnail); + CameraDevice *cameraDevice_; std::unique_ptr encoder_; libcamera::Size streamSize_; diff --git a/src/android/jpeg/thumbnailer.cpp b/src/android/jpeg/thumbnailer.cpp new file mode 100644 index 0000000..3163576 --- /dev/null +++ b/src/android/jpeg/thumbnailer.cpp @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * thumbnailer.cpp - Basic image thumbnailer from NV12 + */ + +#include "thumbnailer.h" + +#include + +#include "libcamera/internal/file.h" +#include "libcamera/internal/log.h" + +using namespace libcamera; + +LOG_DEFINE_CATEGORY(Thumbnailer) + +Thumbnailer::Thumbnailer() + : validConfiguration_(false) +{ +} + +void Thumbnailer::configure(const Size &sourceSize, PixelFormat pixelFormat) +{ + sourceSize_ = sourceSize; + pixelFormat_ = pixelFormat; + + if (pixelFormat_ != formats::NV12) { + LOG (Thumbnailer, Error) << "Failed to configure: Pixel Format " + << pixelFormat_.toString() << " unsupported."; + return; + } + + validConfiguration_ = true; +} + +/* + * The Exif specification recommends the width of the thumbnail to be a + * mutiple of 16 (section 4.8.1). Hence, compute the corresponding height + * keeping the aspect ratio same as of the source. + */ +Size Thumbnailer::computeThumbnailSize() +{ + unsigned int targetHeight; + unsigned int targetWidth = 160; + + targetHeight = targetWidth * sourceSize_.height / sourceSize_.width; + + if (targetHeight & 1) + targetHeight++; + + return Size(targetWidth, targetHeight); +} + +void +Thumbnailer::scaleBuffer(const libcamera::Span &source, + std::vector &destination) +{ + if (!validConfiguration_) { + LOG(Thumbnailer, Error) << "config is unconfigured or invalid."; + return; + } + + targetSize_ = computeThumbnailSize(); + + const unsigned int sw = sourceSize_.width; + const unsigned int sh = sourceSize_.height; + const unsigned int tw = targetSize_.width; + const unsigned int th = targetSize_.height; + + /* Image scaling block implementing nearest-neighbour algorithm. */ + unsigned char *src = static_cast(source.data()); + unsigned char *src_c = src + sh * sw; + unsigned char *src_cb, *src_cr; + + size_t dstSize = (th * tw) + ((th/2) * tw); + destination.reserve(dstSize); + unsigned char *dst = destination.data(); + unsigned char *dst_c = dst + th * tw; + + for (unsigned int y = 0; y < th; y+=2) { + unsigned int sourceY = (sh*y + th/2) / th; + + src_cb = src_c + (sourceY/2) * sw + 0; + src_cr = src_c + (sourceY/2) * sw + 1; + + for (unsigned int x = 0; x < tw; x+=2) { + unsigned int sourceX = (sw*x + tw/2) / tw; + + dst[y * tw + x] = src[sw * sourceY + sourceX]; + dst[(y+1) * tw + x] = src[sw * (sourceY+1) + sourceX]; + dst[y * tw + (x+1)] = src[sw * sourceY + (sourceX+1)]; + dst[(y+1) * tw + (x+1)] = src[sw * (sourceY+1) + (sourceX+1)]; + + dst_c[(y/2) * tw + x + 0] = src_cb[(sourceX/2) * 2]; + dst_c[(y/2) * tw + x + 1] = src_cr[(sourceX/2) * 2]; + } + } +} diff --git a/src/android/jpeg/thumbnailer.h b/src/android/jpeg/thumbnailer.h new file mode 100644 index 0000000..bab9855 --- /dev/null +++ b/src/android/jpeg/thumbnailer.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * thumbnailer.h - Basic image thumbnailer from NV12 + */ +#ifndef __ANDROID_JPEG_THUMBNAILER_H__ +#define __ANDROID_JPEG_THUMBNAILER_H__ + +#include + +#include "libcamera/internal/buffer.h" +#include "libcamera/internal/formats.h" + +class Thumbnailer +{ +public: + Thumbnailer(); + + void configure(const libcamera::Size &sourceSize, + libcamera::PixelFormat pixelFormat); + + /* + * \todo: Discuss if we can return targetSize_ via configure() or + * scaleBuffer(). We need targetSize_ to re-encode the scaled buffer + * via encoder in PostProcssorJpeg::writeThumbnail(). + */ + libcamera::Size computeThumbnailSize(); + void scaleBuffer(const libcamera::Span &source, + std::vector &dest); + +private: + libcamera::PixelFormat pixelFormat_; + libcamera::Size sourceSize_; + libcamera::Size targetSize_; + + bool validConfiguration_; +}; + +#endif /* __ANDROID_JPEG_THUMBNAILER_H__ */ diff --git a/src/android/meson.build b/src/android/meson.build index 5a01bea..3905e2f 100644 --- a/src/android/meson.build +++ b/src/android/meson.build @@ -25,6 +25,7 @@ android_hal_sources = files([ 'jpeg/encoder_libjpeg.cpp', 'jpeg/exif.cpp', 'jpeg/post_processor_jpeg.cpp', + 'jpeg/thumbnailer.cpp', ]) android_camera_metadata_sources = files([