From patchwork Tue Apr 26 09:00:28 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Harvey Yang X-Patchwork-Id: 15716 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 1877CC3256 for ; Tue, 26 Apr 2022 09:01:21 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id B698265650; Tue, 26 Apr 2022 11:01:20 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1650963680; bh=85v4ddmkw5CsLQrMwp41bYY0QRSC/UPx+xtv8L59H9U=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc: From; b=YFjMH1TyHtRPtTBsO4S7qWVCz7uT6ec6G7QEJ+FQYD/w+6M6I4VgQj+3SEGo4Z9lQ BLH4mInxOjM41m0vHq9QLYNVkYgj4w0N7FRVn3j7C3flGsQroTiFsVA2THYuK6He0E O6uo1trbRDELzdLbco/8eVDpB2Ei+HGsliMpPxoP+16O8VrDna/B107sfZJJ4+WlVn Y1lo9jBV1bZuuuvI+DBdfLwH4C8OSI4FR7AmjQxNvLqgDq0sZeovkKdiipe8iUzvX6 rCmFjC5qsHmULtMRePeqgc09x5KUUiq+2tVM6r+6NWFNXf8pihDWScSGyDhoZV/aIY Gal35oEGnHT3w== Received: from mail-pl1-x62c.google.com (mail-pl1-x62c.google.com [IPv6:2607:f8b0:4864:20::62c]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 87B336564D for ; Tue, 26 Apr 2022 11:01:19 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="h67FdW3S"; dkim-atps=neutral Received: by mail-pl1-x62c.google.com with SMTP id n18so29638431plg.5 for ; Tue, 26 Apr 2022 02:01:19 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=JWPwtnCSDf/GMocsIOhTk+B2KQEvYtvW++ofo5vlZ6g=; b=h67FdW3SItd8LedG01ZtW1MRzF6QxW3V8rAQfTOxzta6Y1jN3XQci62hllOhAhES3o c5CUN6n7+KihCEqcU4g/cvoyGNe66qBPDYhP95hQKHcmCcBzEwTsIayPlEbquYwHFwnV xJDcXL++isod6e9ZDMI4e8v+8zoUVbW4TYRKI= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=JWPwtnCSDf/GMocsIOhTk+B2KQEvYtvW++ofo5vlZ6g=; b=rZ7aiw4lfxMfzMhAsfpzBODB6r9pr9Qp1nbwD0pTR7UurIr9BRXa5NvAOfWpFCPajo VR2IquIUOBeDbAMjFi/hzUtBGMiS3UfaKCrzahOQXCzniMO6Sa+C2LNgHyztq39oNmeK 13VxZF+lAoGKZwjfzbPW6D6UYx9fK3CsSUpq9oynHxJHFqa4n8ZoKUDETJpm4Y5jpIgF Fk+aw95n4CjhX8ZuDwos5c3nj59rLVYs6VBiYNZi6Y0g0RQBFWcdBAajMJ+uUPo7KZau EV//6QfYJobZar5EaG4yfvWC4TBdeIIGqov9ix06JfvmZkwwlM/xFAT6F3M6JJfQxgYg 6t/A== X-Gm-Message-State: AOAM533fXNaXHo2lxcvxAvYKjdSoOatSS+qL6zq+11m8ulQmHbOkctrc a0eYIZvVpue5du5NTZxI/1aFNJ+ktGtThGCR X-Google-Smtp-Source: ABdhPJwTDkHKnjOrJF9reOs0RxqTRkySpZd0uZZGCW5Z+F0wZMznhnEvqGczK2U2EsAP29kE4a58uw== X-Received: by 2002:a17:90a:5882:b0:1d9:7f80:de79 with SMTP id j2-20020a17090a588200b001d97f80de79mr9173288pji.231.1650963677629; Tue, 26 Apr 2022 02:01:17 -0700 (PDT) Received: from chenghaoyang.c.googlers.com.com (93.135.240.35.bc.googleusercontent.com. [35.240.135.93]) by smtp.gmail.com with ESMTPSA id c18-20020a056a000ad200b004cdccd3da08sm15449379pfl.44.2022.04.26.02.01.16 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Apr 2022 02:01:17 -0700 (PDT) To: libcamera-devel@lists.libcamera.org Date: Tue, 26 Apr 2022 09:00:28 +0000 Message-Id: <20220426090029.1350103-4-chenghaoyang@chromium.org> X-Mailer: git-send-email 2.36.0.rc2.479.g8af0fa9b8e-goog In-Reply-To: <20220426090029.1350103-1-chenghaoyang@chromium.org> References: <20220426090029.1350103-1-chenghaoyang@chromium.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 3/4] Rework JPEG encoder API and update PostProcessorJpeg and EncoderLibJpeg X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Harvey Yang via libcamera-devel From: Harvey Yang Reply-To: Harvey Yang Cc: Harvey Yang Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" To add JEA in the next CL, this CL reworks JPEG encoder API and move thumbnail encoding code into EncoderLibJpeg's implementation, as JEA supports that as well. --- src/android/jpeg/encoder.h | 9 ++- src/android/jpeg/encoder_libjpeg.cpp | 70 +++++++++++++++++++ src/android/jpeg/encoder_libjpeg.h | 21 +++++- .../jpeg/generic_post_processor_jpeg.cpp | 14 ++++ src/android/jpeg/meson.build | 9 +++ src/android/jpeg/post_processor_jpeg.cpp | 60 ++-------------- src/android/jpeg/post_processor_jpeg.h | 11 +-- src/android/meson.build | 5 +- 8 files changed, 128 insertions(+), 71 deletions(-) create mode 100644 src/android/jpeg/generic_post_processor_jpeg.cpp create mode 100644 src/android/jpeg/meson.build diff --git a/src/android/jpeg/encoder.h b/src/android/jpeg/encoder.h index b974d367..6d527d91 100644 --- a/src/android/jpeg/encoder.h +++ b/src/android/jpeg/encoder.h @@ -12,14 +12,19 @@ #include #include +#include "../camera_request.h" + class Encoder { public: virtual ~Encoder() = default; virtual int configure(const libcamera::StreamConfiguration &cfg) = 0; - virtual int encode(const libcamera::FrameBuffer &source, - libcamera::Span destination, + virtual int encode(Camera3RequestDescriptor::StreamBuffer *streamBuffer, libcamera::Span exifData, unsigned int quality) = 0; + virtual int generateThumbnail(const libcamera::FrameBuffer &source, + const libcamera::Size &targetSize, + unsigned int quality, + std::vector *thumbnail) = 0; }; diff --git a/src/android/jpeg/encoder_libjpeg.cpp b/src/android/jpeg/encoder_libjpeg.cpp index 21a3b33d..b5591e33 100644 --- a/src/android/jpeg/encoder_libjpeg.cpp +++ b/src/android/jpeg/encoder_libjpeg.cpp @@ -24,6 +24,8 @@ #include "libcamera/internal/formats.h" #include "libcamera/internal/mapped_framebuffer.h" +#include "../camera_buffer.h" + using namespace libcamera; LOG_DECLARE_CATEGORY(JPEG) @@ -82,8 +84,17 @@ EncoderLibJpeg::~EncoderLibJpeg() } int EncoderLibJpeg::configure(const StreamConfiguration &cfg) +{ + thumbnailer_.configure(cfg.size, cfg.pixelFormat); + cfg_ = cfg; + + return internalConfigure(cfg); +} + +int EncoderLibJpeg::internalConfigure(const StreamConfiguration &cfg) { const struct JPEGPixelFormatInfo info = findPixelInfo(cfg.pixelFormat); + if (info.colorSpace == JCS_UNKNOWN) return -ENOTSUP; @@ -178,6 +189,65 @@ void EncoderLibJpeg::compressNV(const std::vector> &planes) } } +int EncoderLibJpeg::encode(Camera3RequestDescriptor::StreamBuffer *streamBuffer, + libcamera::Span exifData, + unsigned int quality) +{ + internalConfigure(cfg_); + return encode(*streamBuffer->srcBuffer, streamBuffer->dstBuffer.get()->plane(0), exifData, quality); +} + +int EncoderLibJpeg::generateThumbnail( + const libcamera::FrameBuffer &source, + const libcamera::Size &targetSize, + unsigned int quality, + std::vector *thumbnail) +{ + /* Stores the raw scaled-down thumbnail bytes. */ + std::vector rawThumbnail; + + thumbnailer_.createThumbnail(source, targetSize, &rawThumbnail); + + StreamConfiguration thCfg; + thCfg.size = targetSize; + thCfg.pixelFormat = thumbnailer_.pixelFormat(); + int ret = internalConfigure(thCfg); + + if (!rawThumbnail.empty() && !ret) { + /* + * \todo Avoid value-initialization of all elements of the + * vector. + */ + thumbnail->resize(rawThumbnail.size()); + + /* + * Split planes manually as the encoder expects a vector of + * planes. + * + * \todo Pass a vector of planes directly to + * Thumbnailer::createThumbnailer above and remove the manual + * planes split from here. + */ + std::vector> thumbnailPlanes; + const PixelFormatInfo &formatNV12 = PixelFormatInfo::info(formats::NV12); + size_t yPlaneSize = formatNV12.planeSize(targetSize, 0); + size_t uvPlaneSize = formatNV12.planeSize(targetSize, 1); + thumbnailPlanes.push_back({ rawThumbnail.data(), yPlaneSize }); + thumbnailPlanes.push_back({ rawThumbnail.data() + yPlaneSize, uvPlaneSize }); + + int jpeg_size = encode(thumbnailPlanes, *thumbnail, {}, quality); + thumbnail->resize(jpeg_size); + + LOG(JPEG, Debug) + << "Thumbnail compress returned " + << jpeg_size << " bytes"; + + return jpeg_size; + } + + return -1; +} + int EncoderLibJpeg::encode(const FrameBuffer &source, Span dest, Span exifData, unsigned int quality) { diff --git a/src/android/jpeg/encoder_libjpeg.h b/src/android/jpeg/encoder_libjpeg.h index 1b3ac067..56b27bae 100644 --- a/src/android/jpeg/encoder_libjpeg.h +++ b/src/android/jpeg/encoder_libjpeg.h @@ -15,6 +15,8 @@ #include +#include "thumbnailer.h" + class EncoderLibJpeg : public Encoder { public: @@ -22,19 +24,32 @@ public: ~EncoderLibJpeg(); int configure(const libcamera::StreamConfiguration &cfg) override; + int encode(Camera3RequestDescriptor::StreamBuffer *streamBuffer, + libcamera::Span exifData, + unsigned int quality) override; + int generateThumbnail( + const libcamera::FrameBuffer &source, + const libcamera::Size &targetSize, + unsigned int quality, + std::vector *thumbnail) override; + +private: + int internalConfigure(const libcamera::StreamConfiguration &cfg); + int encode(const libcamera::FrameBuffer &source, libcamera::Span destination, libcamera::Span exifData, - unsigned int quality) override; + unsigned int quality); int encode(const std::vector> &planes, libcamera::Span destination, libcamera::Span exifData, unsigned int quality); -private: void compressRGB(const std::vector> &planes); void compressNV(const std::vector> &planes); + libcamera::StreamConfiguration cfg_; + struct jpeg_compress_struct compress_; struct jpeg_error_mgr jerr_; @@ -42,4 +57,6 @@ private: bool nv_; bool nvSwap_; + + Thumbnailer thumbnailer_; }; diff --git a/src/android/jpeg/generic_post_processor_jpeg.cpp b/src/android/jpeg/generic_post_processor_jpeg.cpp new file mode 100644 index 00000000..890f6972 --- /dev/null +++ b/src/android/jpeg/generic_post_processor_jpeg.cpp @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2022, Google Inc. + * + * generic_post_processor_jpeg.cpp - Generic JPEG Post Processor + */ + +#include "encoder_libjpeg.h" +#include "post_processor_jpeg.h" + +void PostProcessorJpeg::SetEncoder() +{ + encoder_ = std::make_unique(); +} diff --git a/src/android/jpeg/meson.build b/src/android/jpeg/meson.build new file mode 100644 index 00000000..94522dc0 --- /dev/null +++ b/src/android/jpeg/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +android_hal_sources += files([ + 'exif.cpp', + 'post_processor_jpeg.cpp']) + +android_hal_sources += files(['encoder_libjpeg.cpp', + 'generic_post_processor_jpeg.cpp', + 'thumbnailer.cpp']) diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp index d72ebc3c..7ceba60e 100644 --- a/src/android/jpeg/post_processor_jpeg.cpp +++ b/src/android/jpeg/post_processor_jpeg.cpp @@ -9,10 +9,10 @@ #include +#include "../android_framebuffer.h" #include "../camera_device.h" #include "../camera_metadata.h" #include "../camera_request.h" -#include "encoder_libjpeg.h" #include "exif.h" #include @@ -44,60 +44,11 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg, streamSize_ = outCfg.size; - thumbnailer_.configure(inCfg.size, inCfg.pixelFormat); - - encoder_ = std::make_unique(); + SetEncoder(); return encoder_->configure(inCfg); } -void PostProcessorJpeg::generateThumbnail(const FrameBuffer &source, - const Size &targetSize, - unsigned int quality, - std::vector *thumbnail) -{ - /* Stores the raw scaled-down thumbnail bytes. */ - std::vector rawThumbnail; - - thumbnailer_.createThumbnail(source, targetSize, &rawThumbnail); - - StreamConfiguration thCfg; - thCfg.size = targetSize; - thCfg.pixelFormat = thumbnailer_.pixelFormat(); - int ret = thumbnailEncoder_.configure(thCfg); - - if (!rawThumbnail.empty() && !ret) { - /* - * \todo Avoid value-initialization of all elements of the - * vector. - */ - thumbnail->resize(rawThumbnail.size()); - - /* - * Split planes manually as the encoder expects a vector of - * planes. - * - * \todo Pass a vector of planes directly to - * Thumbnailer::createThumbnailer above and remove the manual - * planes split from here. - */ - std::vector> thumbnailPlanes; - const PixelFormatInfo &formatNV12 = PixelFormatInfo::info(formats::NV12); - size_t yPlaneSize = formatNV12.planeSize(targetSize, 0); - size_t uvPlaneSize = formatNV12.planeSize(targetSize, 1); - thumbnailPlanes.push_back({ rawThumbnail.data(), yPlaneSize }); - thumbnailPlanes.push_back({ rawThumbnail.data() + yPlaneSize, uvPlaneSize }); - - int jpeg_size = thumbnailEncoder_.encode(thumbnailPlanes, - *thumbnail, {}, quality); - thumbnail->resize(jpeg_size); - - LOG(JPEG, Debug) - << "Thumbnail compress returned " - << jpeg_size << " bytes"; - } -} - void PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBuffer) { ASSERT(encoder_); @@ -164,8 +115,8 @@ void PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBu if (thumbnailSize != Size(0, 0)) { std::vector thumbnail; - generateThumbnail(source, thumbnailSize, quality, &thumbnail); - if (!thumbnail.empty()) + ret = encoder_->generateThumbnail(source, thumbnailSize, quality, &thumbnail); + if (ret > 0 && !thumbnail.empty()) exif.setThumbnail(thumbnail, Exif::Compression::JPEG); } @@ -194,8 +145,7 @@ void PostProcessorJpeg::process(Camera3RequestDescriptor::StreamBuffer *streamBu const uint8_t quality = ret ? *entry.data.u8 : 95; resultMetadata->addEntry(ANDROID_JPEG_QUALITY, quality); - int jpeg_size = encoder_->encode(source, destination->plane(0), - exif.data(), quality); + int jpeg_size = encoder_->encode(streamBuffer, exif.data(), quality); if (jpeg_size < 0) { LOG(JPEG, Error) << "Failed to encode stream image"; processComplete.emit(streamBuffer, PostProcessor::Status::Error); diff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h index 98309b01..a09f8798 100644 --- a/src/android/jpeg/post_processor_jpeg.h +++ b/src/android/jpeg/post_processor_jpeg.h @@ -8,11 +8,11 @@ #pragma once #include "../post_processor.h" -#include "encoder_libjpeg.h" -#include "thumbnailer.h" #include +#include "encoder.h" + class CameraDevice; class PostProcessorJpeg : public PostProcessor @@ -25,14 +25,9 @@ public: void process(Camera3RequestDescriptor::StreamBuffer *streamBuffer) override; private: - void generateThumbnail(const libcamera::FrameBuffer &source, - const libcamera::Size &targetSize, - unsigned int quality, - std::vector *thumbnail); + void SetEncoder(); CameraDevice *const cameraDevice_; std::unique_ptr encoder_; libcamera::Size streamSize_; - EncoderLibJpeg thumbnailEncoder_; - Thumbnailer thumbnailer_; }; diff --git a/src/android/meson.build b/src/android/meson.build index 27be27bb..026b8b3c 100644 --- a/src/android/meson.build +++ b/src/android/meson.build @@ -48,10 +48,6 @@ android_hal_sources = files([ 'camera_ops.cpp', 'camera_request.cpp', 'camera_stream.cpp', - 'jpeg/encoder_libjpeg.cpp', - 'jpeg/exif.cpp', - 'jpeg/post_processor_jpeg.cpp', - 'jpeg/thumbnailer.cpp', 'yuv/post_processor_yuv.cpp' ]) @@ -59,6 +55,7 @@ android_cpp_args = [] subdir('cros') subdir('mm') +subdir('jpeg') android_camera_metadata_sources = files([ 'metadata/camera_metadata.c',