{"id":10159,"url":"https://patchwork.libcamera.org/api/patches/10159/?format=json","web_url":"https://patchwork.libcamera.org/patch/10159/","project":{"id":1,"url":"https://patchwork.libcamera.org/api/projects/1/?format=json","name":"libcamera","link_name":"libcamera","list_id":"libcamera_core","list_email":"libcamera-devel@lists.libcamera.org","web_url":"","scm_url":"","webscm_url":""},"msgid":"<20201021080806.46636-3-email@uajain.com>","date":"2020-10-21T08:08:06","name":"[libcamera-devel,RFC,2/2] android: jpeg: Add a basic NV12 image thumbnailer","commit_ref":null,"pull_url":null,"state":"superseded","archived":false,"hash":"de47afa9b120b7d6cc1d115e5b2545af0b109ac2","submitter":{"id":1,"url":"https://patchwork.libcamera.org/api/people/1/?format=json","name":"Umang Jain","email":"email@uajain.com"},"delegate":{"id":12,"url":"https://patchwork.libcamera.org/api/users/12/?format=json","username":"uajain","first_name":"Umang","last_name":"Jain","email":"umang.jain@ideasonboard.com"},"mbox":"https://patchwork.libcamera.org/patch/10159/mbox/","series":[{"id":1397,"url":"https://patchwork.libcamera.org/api/series/1397/?format=json","web_url":"https://patchwork.libcamera.org/project/libcamera/list/?series=1397","date":"2020-10-21T08:08:04","name":"andriod: Basic NV12 thumbnailer","version":1,"mbox":"https://patchwork.libcamera.org/series/1397/mbox/"}],"comments":"https://patchwork.libcamera.org/api/patches/10159/comments/","check":"pending","checks":"https://patchwork.libcamera.org/api/patches/10159/checks/","tags":{},"headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6998EC3D3C\n\tfor ;\n\tWed, 21 Oct 2020 08:08:22 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3728561DCA;\n\tWed, 21 Oct 2020 10:08:22 +0200 (CEST)","from mail.uajain.com (static.126.159.217.95.clients.your-server.de\n\t[95.217.159.126])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id AEDDA60361\n\tfor ;\n\tWed, 21 Oct 2020 10:08:20 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=uajain.com header.i=@uajain.com\n\theader.b=\"giMKzV8u\"; dkim-atps=neutral","From":"Umang Jain ","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=uajain.com; s=mail;\n\tt=1603267700; bh=M0rdj81Gq/K9cAEN3s3eNXgUN4lZ8Uuw8RxwvtSJ9H4=;\n\th=From:To:Cc:Subject:In-Reply-To:References;\n\tb=giMKzV8u0X+I9TYHzkdRM5WyEjlF2DiA1axEwykhOxdHJEZ5LTm5ln1ZzO3k5gmiL\n\tnExOge8s3X+5fD/F33g0BhEf6l7PdKt4xScuuPrbf92JhGRYyj/hKOuxAKErMWdDjM\n\tGoOb3D4+t03yX8z1sEiFQeWZBccioQYHVu5FVuOjtx0SKTleUiQgaPn748D9ULKODk\n\tVzFfhO++aWM3r6Yu69Jy0LAoZHMIXIv6IdBcnERpZjtwbOW0cJm5lZn66osgBrgbqA\n\tm4v564xoSqqRPTtv4mCz7yjeBazSshD028oQjIlPT2fJQR3ZBNdwAl7BhB6cTZ3sEN\n\ta85tvKslClCLw==","To":"libcamera-devel@lists.libcamera.org","Date":"Wed, 21 Oct 2020 13:38:06 +0530","Message-Id":"<20201021080806.46636-3-email@uajain.com>","In-Reply-To":"<20201021080806.46636-1-email@uajain.com>","References":"<20201021080806.46636-1-email@uajain.com>","Mime-Version":"1.0","Subject":"[libcamera-devel] [RFC PATCH 2/2] android: jpeg: Add a basic NV12\n\timage thumbnailer","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Content-Type":"text/plain; charset=\"us-ascii\"","Content-Transfer-Encoding":"7bit","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "},"content":"Add a basic image thumbnailer for NV12 frames being captured.\nIt shall generate a thumbnail image to be embedded as a part of\nEXIF metadata of the frame. The output of the thumbnail will still\nbe NV12.\n\nSigned-off-by: Umang Jain \n---\n src/android/jpeg/post_processor_jpeg.cpp | 39 +++++++++\n src/android/jpeg/post_processor_jpeg.h | 3 +\n src/android/jpeg/thumbnailer.cpp | 100 +++++++++++++++++++++++\n src/android/jpeg/thumbnailer.h | 40 +++++++++\n src/android/meson.build | 1 +\n 5 files changed, 183 insertions(+)\n create mode 100644 src/android/jpeg/thumbnailer.cpp\n create mode 100644 src/android/jpeg/thumbnailer.h","diff":"diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp\nindex 9d452b7..f5f1f78 100644\n--- a/src/android/jpeg/post_processor_jpeg.cpp\n+++ b/src/android/jpeg/post_processor_jpeg.cpp\n@@ -11,6 +11,7 @@\n #include \"../camera_metadata.h\"\n #include \"encoder_libjpeg.h\"\n #include \"exif.h\"\n+#include \"thumbnailer.h\"\n \n #include \n \n@@ -44,6 +45,32 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg,\n \treturn encoder_->configure(inCfg);\n }\n \n+void PostProcessorJpeg::generateThumbnail(const libcamera::Span &source,\n+\t\t\t\t\t std::vector &thumbnail)\n+{\n+\tlibcamera::Span destination;\n+\tThumbnailer thumbnailer;\n+\n+\tthumbnailer.configure(streamSize_, formats::NV12);\n+\tlibcamera::Size targetSize = thumbnailer.computeThumbnailSize();\n+\tthumbnailer.scaleBuffer(source, thumbnail);\n+\n+\tif (thumbnail.data()) {\n+\t\tStreamConfiguration thumbnailCfg;\n+\n+\t\tstd::unique_ptr encoder =\n+\t\t\t\tstd::make_unique();\n+\n+\t\tthumbnailCfg.pixelFormat = formats::NV12;\n+\t\tthumbnailCfg.size = targetSize;\n+\t\tencoder->configure(thumbnailCfg);\n+\t\tint jpeg_size = encoder->encode({ thumbnail.data(), thumbnail.capacity() },\n+\t\t\t\tdestination, { });\n+\t\tLOG(JPEG, Info) << \"Thumbnail compress returned \"\n+\t\t\t\t<< jpeg_size << \" bytes\";\n+\t}\n+}\n+\n int PostProcessorJpeg::process(const libcamera::FrameBuffer *source,\n \t\t\t const libcamera::Span &destination,\n \t\t\t CameraMetadata *metadata)\n@@ -73,6 +100,18 @@ int PostProcessorJpeg::process(const libcamera::FrameBuffer *source,\n \t\treturn jpeg_size;\n \t}\n \n+\tstd::vector thumbnail;\n+\tgenerateThumbnail(destination, thumbnail);\n+\t/*\n+\t * \\todo: Write the compressed thumbnail to a file for inspection.\n+\t * (I) Check if we can still write the thumbnail to EXIF here.\n+\t * If not, we might need to move the thumbnailer logic to encoder.\n+\t * And if we do that, first we need to make sure we get can\n+\t * compressed data written to destination first before calling\n+\t * jpeg_finish_compress operation somehow. Thumbnailing will\n+\t * only occur if we have compressed data available first.\n+\t */\n+\n \t/*\n \t * Fill in the JPEG blob header.\n \t *\ndiff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h\nindex 62c8650..05601ee 100644\n--- a/src/android/jpeg/post_processor_jpeg.h\n+++ b/src/android/jpeg/post_processor_jpeg.h\n@@ -28,6 +28,9 @@ public:\n \t\t CameraMetadata *metadata) override;\n \n private:\n+\tvoid generateThumbnail(const libcamera::Span &source,\n+\t\t\t std::vector &thumbnail);\n+\n \tCameraDevice *cameraDevice_;\n \tstd::unique_ptr encoder_;\n \tlibcamera::Size streamSize_;\ndiff --git a/src/android/jpeg/thumbnailer.cpp b/src/android/jpeg/thumbnailer.cpp\nnew file mode 100644\nindex 0000000..3163576\n--- /dev/null\n+++ b/src/android/jpeg/thumbnailer.cpp\n@@ -0,0 +1,100 @@\n+/* SPDX-License-Identifier: GPL-2.0-or-later */\n+/*\n+ * Copyright (C) 2020, Google Inc.\n+ *\n+ * thumbnailer.cpp - Basic image thumbnailer from NV12\n+ */\n+\n+#include \"thumbnailer.h\"\n+\n+#include \n+\n+#include \"libcamera/internal/file.h\"\n+#include \"libcamera/internal/log.h\"\n+\n+using namespace libcamera;\n+\n+LOG_DEFINE_CATEGORY(Thumbnailer)\n+\n+Thumbnailer::Thumbnailer()\n+\t: validConfiguration_(false)\n+{\n+}\n+\n+void Thumbnailer::configure(const Size &sourceSize, PixelFormat pixelFormat)\n+{\n+\tsourceSize_ = sourceSize;\n+\tpixelFormat_ = pixelFormat;\n+\n+\tif (pixelFormat_ != formats::NV12) {\n+\t\tLOG (Thumbnailer, Error) << \"Failed to configure: Pixel Format \"\n+\t\t\t\t << pixelFormat_.toString() << \" unsupported.\";\n+\t\treturn;\n+\t}\n+\n+\tvalidConfiguration_ = true;\n+}\n+\n+/*\n+ * The Exif specification recommends the width of the thumbnail to be a\n+ * mutiple of 16 (section 4.8.1). Hence, compute the corresponding height\n+ * keeping the aspect ratio same as of the source.\n+ */\n+Size Thumbnailer::computeThumbnailSize()\n+{\n+\tunsigned int targetHeight;\n+\tunsigned int targetWidth = 160;\n+\n+\ttargetHeight = targetWidth * sourceSize_.height / sourceSize_.width;\n+\n+\tif (targetHeight & 1)\n+\t\ttargetHeight++;\n+\n+\treturn Size(targetWidth, targetHeight);\n+}\n+\n+void\n+Thumbnailer::scaleBuffer(const libcamera::Span &source,\n+\t\t\t std::vector &destination)\n+{\n+\tif (!validConfiguration_) {\n+\t\tLOG(Thumbnailer, Error) << \"config is unconfigured or invalid.\";\n+\t\treturn;\n+\t}\n+\n+\ttargetSize_ = computeThumbnailSize();\n+\n+\tconst unsigned int sw = sourceSize_.width;\n+\tconst unsigned int sh = sourceSize_.height;\n+\tconst unsigned int tw = targetSize_.width;\n+\tconst unsigned int th = targetSize_.height;\n+\n+\t/* Image scaling block implementing nearest-neighbour algorithm. */\n+\tunsigned char *src = static_cast(source.data());\n+\tunsigned char *src_c = src + sh * sw;\n+\tunsigned char *src_cb, *src_cr;\n+\n+\tsize_t dstSize = (th * tw) + ((th/2) * tw);\n+\tdestination.reserve(dstSize);\n+\tunsigned char *dst = destination.data();\n+\tunsigned char *dst_c = dst + th * tw;\n+\n+\tfor (unsigned int y = 0; y < th; y+=2) {\n+\t\tunsigned int sourceY = (sh*y + th/2) / th;\n+\n+\t\tsrc_cb = src_c + (sourceY/2) * sw + 0;\n+\t\tsrc_cr = src_c + (sourceY/2) * sw + 1;\n+\n+\t\tfor (unsigned int x = 0; x < tw; x+=2) {\n+\t\t\tunsigned int sourceX = (sw*x + tw/2) / tw;\n+\n+\t\t\tdst[y * tw + x] = src[sw * sourceY + sourceX];\n+\t\t\tdst[(y+1) * tw + x] = src[sw * (sourceY+1) + sourceX];\n+\t\t\tdst[y * tw + (x+1)] = src[sw * sourceY + (sourceX+1)];\n+\t\t\tdst[(y+1) * tw + (x+1)] = src[sw * (sourceY+1) + (sourceX+1)];\n+\n+\t\t\tdst_c[(y/2) * tw + x + 0] = src_cb[(sourceX/2) * 2];\n+\t\t\tdst_c[(y/2) * tw + x + 1] = src_cr[(sourceX/2) * 2];\n+\t\t}\n+\t}\n+}\ndiff --git a/src/android/jpeg/thumbnailer.h b/src/android/jpeg/thumbnailer.h\nnew file mode 100644\nindex 0000000..bab9855\n--- /dev/null\n+++ b/src/android/jpeg/thumbnailer.h\n@@ -0,0 +1,40 @@\n+/* SPDX-License-Identifier: GPL-2.0-or-later */\n+/*\n+ * Copyright (C) 2020, Google Inc.\n+ *\n+ * thumbnailer.h - Basic image thumbnailer from NV12\n+ */\n+#ifndef __ANDROID_JPEG_THUMBNAILER_H__\n+#define __ANDROID_JPEG_THUMBNAILER_H__\n+\n+#include \n+\n+#include \"libcamera/internal/buffer.h\"\n+#include \"libcamera/internal/formats.h\"\n+\n+class Thumbnailer\n+{\n+public:\n+\tThumbnailer();\n+\n+\tvoid configure(const libcamera::Size &sourceSize,\n+\t\t libcamera::PixelFormat pixelFormat);\n+\n+\t/*\n+\t * \\todo: Discuss if we can return targetSize_ via configure() or\n+\t * scaleBuffer(). We need targetSize_ to re-encode the scaled buffer\n+\t * via encoder in PostProcssorJpeg::writeThumbnail().\n+\t */\n+\tlibcamera::Size computeThumbnailSize();\n+\tvoid scaleBuffer(const libcamera::Span &source,\n+\t\t\t std::vector &dest);\n+\n+private:\n+\tlibcamera::PixelFormat pixelFormat_;\n+\tlibcamera::Size sourceSize_;\n+\tlibcamera::Size targetSize_;\n+\n+\tbool validConfiguration_;\n+};\n+\n+#endif /* __ANDROID_JPEG_THUMBNAILER_H__ */\ndiff --git a/src/android/meson.build b/src/android/meson.build\nindex 5a01bea..3905e2f 100644\n--- a/src/android/meson.build\n+++ b/src/android/meson.build\n@@ -25,6 +25,7 @@ android_hal_sources = files([\n 'jpeg/encoder_libjpeg.cpp',\n 'jpeg/exif.cpp',\n 'jpeg/post_processor_jpeg.cpp',\n+ 'jpeg/thumbnailer.cpp',\n ])\n \n android_camera_metadata_sources = files([\n","prefixes":["libcamera-devel","RFC","2/2"]}