From patchwork Thu Aug 6 14:50:18 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Kieran Bingham X-Patchwork-Id: 9261 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id E4D70BD86F for ; Thu, 6 Aug 2020 14:50:26 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 23F4D60DFC; Thu, 6 Aug 2020 16:50:26 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="YuWhFJsP"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id F1E6460495 for ; Thu, 6 Aug 2020 16:50:23 +0200 (CEST) Received: from Q.local (cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 694FE560; Thu, 6 Aug 2020 16:50:23 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1596725423; bh=uvlVs9VWG6TRorU591xgId2eFIlc5P2N9YkIsOMlTQg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=YuWhFJsP9pbNefc0fcXBuXPmii+Yvpw9KBAAPztlVJ5UyUtQdIHRwX+cYE0iRFksK ukq/hZFL5e8Bk/xursam8K/hzlkIgSR0D0FOcrTwXInCnPeTH1S5fLypxYYoFlqQoH jCom6OA7ImO5eJuxKz5c8FR22SWvCrj2RR8FUw7U= From: Kieran Bingham To: libcamera devel Date: Thu, 6 Aug 2020 15:50:18 +0100 Message-Id: <20200806145019.290878-2-kieran.bingham@ideasonboard.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200806145019.290878-1-kieran.bingham@ideasonboard.com> References: <20200805151507.227503-1-kieran.bingham@ideasonboard.com> <20200806145019.290878-1-kieran.bingham@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 1/2] android: Introduce JPEG encoding X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Provide an encoder interface and implement a JPEG encoder using libjpeg. Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/android/jpeg/encoder.h | 24 +++ src/android/jpeg/encoder_libjpeg.cpp | 218 +++++++++++++++++++++++++++ src/android/jpeg/encoder_libjpeg.h | 42 ++++++ src/android/meson.build | 5 + src/libcamera/meson.build | 2 + 5 files changed, 291 insertions(+) create mode 100644 src/android/jpeg/encoder.h create mode 100644 src/android/jpeg/encoder_libjpeg.cpp create mode 100644 src/android/jpeg/encoder_libjpeg.h diff --git a/src/android/jpeg/encoder.h b/src/android/jpeg/encoder.h new file mode 100644 index 000000000000..f9eb88e65c1d --- /dev/null +++ b/src/android/jpeg/encoder.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * encoder.h - Image encoding interface + */ +#ifndef __ANDROID_JPEG_ENCODER_H__ +#define __ANDROID_JPEG_ENCODER_H__ + +#include +#include +#include + +class Encoder +{ +public: + virtual ~Encoder() {}; + + virtual int configure(const libcamera::StreamConfiguration &cfg) = 0; + virtual int encode(const libcamera::FrameBuffer *source, + const libcamera::Span &destination) = 0; +}; + +#endif /* __ANDROID_JPEG_ENCODER_H__ */ diff --git a/src/android/jpeg/encoder_libjpeg.cpp b/src/android/jpeg/encoder_libjpeg.cpp new file mode 100644 index 000000000000..977538ce6f75 --- /dev/null +++ b/src/android/jpeg/encoder_libjpeg.cpp @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * encoder_libjpeg.cpp - JPEG encoding using libjpeg native API + */ + +#include "encoder_libjpeg.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "libcamera/internal/formats.h" +#include "libcamera/internal/log.h" + +using namespace libcamera; + +LOG_DEFINE_CATEGORY(JPEG) + +namespace { + +struct JPEGPixelFormatInfo { + J_COLOR_SPACE colorSpace; + const PixelFormatInfo &pixelFormatInfo; + bool nvSwap; +}; + +const std::map pixelInfo{ + { formats::R8, { JCS_GRAYSCALE, PixelFormatInfo::info(formats::R8), false } }, + + { formats::RGB888, { JCS_EXT_BGR, PixelFormatInfo::info(formats::RGB888), false } }, + { formats::BGR888, { JCS_EXT_RGB, PixelFormatInfo::info(formats::BGR888), false } }, + + { formats::NV12, { JCS_YCbCr, PixelFormatInfo::info(formats::NV12), false } }, + { formats::NV21, { JCS_YCbCr, PixelFormatInfo::info(formats::NV21), true } }, + { formats::NV16, { JCS_YCbCr, PixelFormatInfo::info(formats::NV16), false } }, + { formats::NV61, { JCS_YCbCr, PixelFormatInfo::info(formats::NV61), true } }, + { formats::NV24, { JCS_YCbCr, PixelFormatInfo::info(formats::NV24), false } }, + { formats::NV42, { JCS_YCbCr, PixelFormatInfo::info(formats::NV42), true } }, +}; + +const struct JPEGPixelFormatInfo &findPixelInfo(const PixelFormat &format) +{ + static const struct JPEGPixelFormatInfo invalidPixelFormat { + JCS_UNKNOWN, PixelFormatInfo(), false + }; + + const auto iter = pixelInfo.find(format); + if (iter == pixelInfo.end()) { + LOG(JPEG, Error) << "Unsupported pixel format for JPEG encoder: " + << format.toString(); + return invalidPixelFormat; + } + + return iter->second; +} + +} /* namespace */ + +EncoderLibJpeg::EncoderLibJpeg() + : quality_(95) +{ + /* \todo Expand error handling coverage with a custom handler. */ + compress_.err = jpeg_std_error(&jerr_); + + jpeg_create_compress(&compress_); +} + +EncoderLibJpeg::~EncoderLibJpeg() +{ + jpeg_destroy_compress(&compress_); +} + +int EncoderLibJpeg::configure(const StreamConfiguration &cfg) +{ + const struct JPEGPixelFormatInfo info = findPixelInfo(cfg.pixelFormat); + if (info.colorSpace == JCS_UNKNOWN) + return -ENOTSUP; + + compress_.image_width = cfg.size.width; + compress_.image_height = cfg.size.height; + compress_.in_color_space = info.colorSpace; + + compress_.input_components = info.colorSpace == JCS_GRAYSCALE ? 1 : 3; + + jpeg_set_defaults(&compress_); + jpeg_set_quality(&compress_, quality_, TRUE); + + pixelFormatInfo_ = &info.pixelFormatInfo; + + nv_ = pixelFormatInfo_->numPlanes() == 2; + nvSwap_ = info.nvSwap; + + return 0; +} + +void EncoderLibJpeg::compressRGB(const libcamera::MappedBuffer *frame) +{ + unsigned char *src = static_cast(frame->maps()[0].data()); + /* \todo Stride information should come from buffer configuration. */ + unsigned int stride = pixelFormatInfo_->stride(compress_.image_width, 0); + + JSAMPROW row_pointer[1]; + + while (compress_.next_scanline < compress_.image_height) { + row_pointer[0] = &src[compress_.next_scanline * stride]; + jpeg_write_scanlines(&compress_, row_pointer, 1); + } +} + +/* + * Compress the incoming buffer from a supported NV format. + * This naively unpacks the semi-planar NV12 to a YUV888 format for libjpeg. + */ +void EncoderLibJpeg::compressNV(const libcamera::MappedBuffer *frame) +{ + uint8_t tmprowbuf[compress_.image_width * 3]; + + /* + * \todo Use the raw api, and only unpack the cb/cr samples to new line + * buffers. If possible, see if we can set appropriate pixel strides + * too to save even that copy. + * + * Possible hints at: + * https://sourceforge.net/p/libjpeg/mailman/message/30815123/ + */ + unsigned int y_stride = pixelFormatInfo_->stride(compress_.image_width, 0); + unsigned int c_stride = pixelFormatInfo_->stride(compress_.image_width, 1); + + unsigned int horzSubSample = 2 * compress_.image_width / c_stride; + unsigned int vertSubSample = pixelFormatInfo_->planes[1].verticalSubSampling; + + unsigned int c_inc = horzSubSample == 1 ? 2 : 0; + unsigned int cb_pos = nvSwap_ ? 1 : 0; + unsigned int cr_pos = nvSwap_ ? 0 : 1; + + const unsigned char *src = static_cast(frame->maps()[0].data()); + const unsigned char *src_c = src + y_stride * compress_.image_height; + + JSAMPROW row_pointer[1]; + row_pointer[0] = &tmprowbuf[0]; + + for (unsigned int y = 0; y < compress_.image_height; y++) { + unsigned char *dst = &tmprowbuf[0]; + + const unsigned char *src_y = src + y * compress_.image_width; + const unsigned char *src_cb = src_c + (y / vertSubSample) * c_stride + cb_pos; + const unsigned char *src_cr = src_c + (y / vertSubSample) * c_stride + cr_pos; + + for (unsigned int x = 0; x < compress_.image_width; x += 2) { + dst[0] = *src_y; + dst[1] = *src_cb; + dst[2] = *src_cr; + src_y++; + src_cb += c_inc; + src_cr += c_inc; + dst += 3; + + dst[0] = *src_y; + dst[1] = *src_cb; + dst[2] = *src_cr; + src_y++; + src_cb += 2; + src_cr += 2; + dst += 3; + } + + jpeg_write_scanlines(&compress_, row_pointer, 1); + } +} + +int EncoderLibJpeg::encode(const FrameBuffer *source, + const libcamera::Span &dest) +{ + MappedFrameBuffer frame(source, PROT_READ); + if (!frame.isValid()) { + LOG(JPEG, Error) << "Failed to map FrameBuffer : " + << strerror(frame.error()); + return frame.error(); + } + + unsigned char *destination = dest.data(); + unsigned long size = dest.size(); + + /* + * The jpeg_mem_dest will reallocate if the required size is not + * sufficient. That means the output won't be written to the correct + * buffers. + * + * \todo Implement our own custom memory destination to prevent + * reallocation and prefer failure with correct reporting. + */ + jpeg_mem_dest(&compress_, &destination, &size); + + jpeg_start_compress(&compress_, TRUE); + + LOG(JPEG, Debug) << "JPEG Encode Starting:" << compress_.image_width + << "x" << compress_.image_height; + + if (nv_) + compressNV(&frame); + else + compressRGB(&frame); + + jpeg_finish_compress(&compress_); + + return size; +} diff --git a/src/android/jpeg/encoder_libjpeg.h b/src/android/jpeg/encoder_libjpeg.h new file mode 100644 index 000000000000..aed081a46b99 --- /dev/null +++ b/src/android/jpeg/encoder_libjpeg.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * encoder_libjpeg.h - JPEG encoding using libjpeg + */ +#ifndef __ANDROID_JPEG_ENCODER_LIBJPEG_H__ +#define __ANDROID_JPEG_ENCODER_LIBJPEG_H__ + +#include "encoder.h" + +#include "libcamera/internal/buffer.h" +#include "libcamera/internal/formats.h" + +#include + +class EncoderLibJpeg : public Encoder +{ +public: + EncoderLibJpeg(); + ~EncoderLibJpeg(); + + int configure(const libcamera::StreamConfiguration &cfg) override; + int encode(const libcamera::FrameBuffer *source, + const libcamera::Span &destination) override; + +private: + void compressRGB(const libcamera::MappedBuffer *frame); + void compressNV(const libcamera::MappedBuffer *frame); + + struct jpeg_compress_struct compress_; + struct jpeg_error_mgr jerr_; + + unsigned int quality_; + + const libcamera::PixelFormatInfo *pixelFormatInfo_; + + bool nv_; + bool nvSwap_; +}; + +#endif /* __ANDROID_JPEG_ENCODER_LIBJPEG_H__ */ diff --git a/src/android/meson.build b/src/android/meson.build index 822cad621f01..f7b81a498499 100644 --- a/src/android/meson.build +++ b/src/android/meson.build @@ -6,12 +6,17 @@ android_hal_sources = files([ 'camera_device.cpp', 'camera_metadata.cpp', 'camera_ops.cpp', + 'jpeg/encoder_libjpeg.cpp', ]) android_camera_metadata_sources = files([ 'metadata/camera_metadata.c', ]) +android_deps = [ + dependency('libjpeg'), +] + android_camera_metadata = static_library('camera_metadata', android_camera_metadata_sources, include_directories : android_includes) diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index bada45bcb11e..af2f3d950be6 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -125,6 +125,8 @@ if get_option('android') libcamera_sources += android_hal_sources includes += android_includes libcamera_link_with += android_camera_metadata + + libcamera_deps += android_deps endif # We add '/' to the build_rpath as a 'safe' path to act as a boolean flag. From patchwork Thu Aug 6 14:50:19 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Kieran Bingham X-Patchwork-Id: 9262 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 57A9BBD87B for ; Thu, 6 Aug 2020 14:50:27 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 7254C60E00; Thu, 6 Aug 2020 16:50:26 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="LE47bPDL"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 40E4D60495 for ; Thu, 6 Aug 2020 16:50:24 +0200 (CEST) Received: from Q.local (cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id C877BAC8; Thu, 6 Aug 2020 16:50:23 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1596725424; bh=kVNZom8kenDkpWcC43tpJPOmb/RVwkp0GiwaZk0GNaY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=LE47bPDLrwYFmTIiwf69+VTV+/+DyX4zFMMeKx/mTnHl1mjHd1rl1k4YgOCtHEVnk g1TojxdkE5LauYiqy9B6wq91qTmbJprHQirtmdKMEviW2zUDgq2jlOlgM1KIInISiq DNeAsKK30UFI+6n7G7a7V2FV0XRcMCgDBFjT0AWE= From: Kieran Bingham To: libcamera devel Date: Thu, 6 Aug 2020 15:50:19 +0100 Message-Id: <20200806145019.290878-3-kieran.bingham@ideasonboard.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200806145019.290878-1-kieran.bingham@ideasonboard.com> References: <20200805151507.227503-1-kieran.bingham@ideasonboard.com> <20200806145019.290878-1-kieran.bingham@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 2/2] android: camera_device: Support MJPEG stream construction X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" MJPEG streams must be created referencing a libcamera stream. This stream may already be provided by the request configuration, in which case the existing stream is utilised. If no compatible stream is available to encode, a new stream is requested from the libcamera configuration. Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/android/camera_device.cpp | 230 ++++++++++++++++++++++++++++++++-- src/android/camera_device.h | 12 ++ 2 files changed, 233 insertions(+), 9 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index e78fecf46f8d..d9183501228f 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,7 @@ #include "camera_device.h" #include "camera_ops.h" +#include #include #include @@ -22,6 +23,8 @@ #include "camera_metadata.h" #include "system/graphics.h" +#include "jpeg/encoder_libjpeg.h" + using namespace libcamera; namespace { @@ -129,6 +132,52 @@ const std::map camera3FormatsMap = { LOG_DECLARE_CATEGORY(HAL); +class MappedCamera3Buffer : public MappedBuffer +{ +public: + MappedCamera3Buffer(const buffer_handle_t camera3buffer, int flags); +}; + +MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer, + int flags) +{ + maps_.reserve(camera3buffer->numFds); + error_ = 0; + + for (int i = 0; i < camera3buffer->numFds; i++) { + if (camera3buffer->data[i] == -1) + continue; + + off_t length = lseek(camera3buffer->data[i], 0, SEEK_END); + if (length < 0) { + error_ = -errno; + LOG(HAL, Error) << "Failed to query plane length"; + break; + } + + void *address = mmap(nullptr, length, flags, MAP_SHARED, + camera3buffer->data[i], 0); + if (address == MAP_FAILED) { + error_ = -errno; + LOG(HAL, Error) << "Failed to mmap plane"; + break; + } + + maps_.emplace_back(static_cast(address), + static_cast(length)); + } +} + +CameraStream::CameraStream(PixelFormat f, Size s) + : index(-1), format(f), size(s), jpeg(nullptr) +{ +} + +CameraStream::~CameraStream() +{ + delete jpeg; +}; + /* * \struct Camera3RequestDescriptor * @@ -167,6 +216,12 @@ CameraDevice::CameraDevice(unsigned int id, const std::shared_ptr &camer facing_(CAMERA_FACING_FRONT), orientation_(0) { camera_->requestCompleted.connect(this, &CameraDevice::requestComplete); + + /* + * \todo Determine a more accurate value for this during + * streamConfiguration. + */ + maxJpegBufferSize_ = 13 << 20; /* 13631488 from USB HAL */ } CameraDevice::~CameraDevice() @@ -417,10 +472,10 @@ std::tuple CameraDevice::calculateStaticMetadataSize() { /* * \todo Keep this in sync with the actual number of entries. - * Currently: 50 entries, 671 bytes of static metadata + * Currently: 51 entries, 687 bytes of static metadata */ - uint32_t numEntries = 50; - uint32_t byteSize = 671; + uint32_t numEntries = 51; + uint32_t byteSize = 687; /* * Calculate space occupation in bytes for dynamically built metadata @@ -576,6 +631,12 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() availableThumbnailSizes.data(), availableThumbnailSizes.size()); + /* + * \todo Calculate the maximum JPEG buffer size by asking the encoder + * giving the maximum frame size required. + */ + staticMetadata_->addEntry(ANDROID_JPEG_MAX_SIZE, &maxJpegBufferSize_, 1); + /* Sensor static metadata. */ int32_t pixelArraySize[] = { 2592, 1944, @@ -789,6 +850,7 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() ANDROID_CONTROL_AWB_LOCK_AVAILABLE, ANDROID_CONTROL_AVAILABLE_MODES, ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES, + ANDROID_JPEG_MAX_SIZE, ANDROID_SENSOR_INFO_PIXEL_ARRAY_SIZE, ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE, ANDROID_SENSOR_INFO_SENSITIVITY_RANGE, @@ -860,6 +922,9 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() ANDROID_SENSOR_EXPOSURE_TIME, ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, ANDROID_STATISTICS_SCENE_FLICKER, + ANDROID_JPEG_SIZE, + ANDROID_JPEG_QUALITY, + ANDROID_JPEG_ORIENTATION, }; staticMetadata_->addEntry(ANDROID_REQUEST_AVAILABLE_RESULT_KEYS, availableResultKeys.data(), @@ -1052,8 +1117,10 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) */ unsigned int streamIndex = 0; + /* First handle all non-MJPEG streams. */ for (unsigned int i = 0; i < stream_list->num_streams; ++i) { camera3_stream_t *stream = stream_list->streams[i]; + Size size(stream->width, stream->height); PixelFormat format = toPixelFormat(stream->format); @@ -1067,16 +1134,71 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) if (!format.isValid()) return -EINVAL; + streams_.emplace_back(format, size); + stream->priv = static_cast(&streams_[i]); + + /* Defer handling of MJPEG streams until all others are known. */ + if (format == formats::MJPEG) + continue; + StreamConfiguration streamConfiguration; - streamConfiguration.size.width = stream->width; - streamConfiguration.size.height = stream->height; + streamConfiguration.size = size; streamConfiguration.pixelFormat = format; config_->addConfiguration(streamConfiguration); streams_[i].index = streamIndex++; - stream->priv = static_cast(&streams_[i]); + } + + /* Now handle MJPEG streams, adding a new stream if required. */ + for (unsigned int i = 0; i < stream_list->num_streams; ++i) { + camera3_stream_t *stream = stream_list->streams[i]; + bool match = false; + + if (streams_[i].format != formats::MJPEG) + continue; + + /* Search for a compatible stream */ + for (unsigned int j = 0; j < config_->size(); j++) { + StreamConfiguration &cfg = config_->at(j); + + /* + * \todo The PixelFormat must also be compatible with + * the encoder. + */ + if (cfg.size == streams_[i].size) { + LOG(HAL, Info) << "Stream " << i + << " using libcamera stream " << j; + + match = true; + streams_[i].index = j; + } + } + + /* + * Without a compatible match for JPEG encoding we must + * introduce a new stream to satisfy the request requirements. + */ + if (!match) { + StreamConfiguration streamConfiguration; + + /* + * \todo The pixelFormat should be a 'best-fit' choice + * and may require a validation cycle. This is not yet + * handled, and should be considered as part of any + * stream configuration reworks. + */ + streamConfiguration.size.width = stream->width; + streamConfiguration.size.height = stream->height; + streamConfiguration.pixelFormat = formats::NV12; + + LOG(HAL, Info) << "Adding " << streamConfiguration.toString() + << " for MJPEG support"; + + config_->addConfiguration(streamConfiguration); + streams_[i].index = streamIndex++; + } } switch (config_->validate()) { @@ -1103,6 +1225,20 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) /* Use the bufferCount confirmed by the validation process. */ stream->max_buffers = cfg.bufferCount; + + /* + * Construct a software encoder for MJPEG streams from the + * chosen libcamera source stream. + */ + if (cameraStream->format == formats::MJPEG) { + cameraStream->jpeg = new EncoderLibJpeg(); + int ret = cameraStream->jpeg->configure(cfg); + if (ret) { + LOG(HAL, Error) + << "Failed to configure encoder"; + return ret; + } + } } /* @@ -1197,6 +1333,10 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques descriptor->buffers[i].stream = camera3Buffers[i].stream; descriptor->buffers[i].buffer = camera3Buffers[i].buffer; + /* Software streams are handled after hardware streams complete. */ + if (cameraStream->format == formats::MJPEG) + continue; + /* * Create a libcamera buffer using the dmabuf descriptors of * the camera3Buffer for each stream. The FrameBuffer is @@ -1253,8 +1393,80 @@ void CameraDevice::requestComplete(Request *request) resultMetadata = getResultMetadata(descriptor->frameNumber, buffer->metadata().timestamp); - /* Prepare to call back the Android camera stack. */ + /* Handle any JPEG compression. */ + for (unsigned int i = 0; i < descriptor->numBuffers; ++i) { + CameraStream *cameraStream = + static_cast(descriptor->buffers[i].stream->priv); + + if (cameraStream->format != formats::MJPEG) + continue; + + Encoder *encoder = cameraStream->jpeg; + if (!encoder) { + LOG(HAL, Error) << "Failed to identify encoder"; + continue; + } + StreamConfiguration *streamConfiguration = &config_->at(cameraStream->index); + Stream *stream = streamConfiguration->stream(); + FrameBuffer *buffer = request->findBuffer(stream); + if (!buffer) { + LOG(HAL, Error) << "Failed to find a source stream buffer"; + continue; + } + + /* + * \todo Buffer mapping and compression should be moved to a + * separate thread. + */ + + MappedCamera3Buffer mapped(*descriptor->buffers[i].buffer, + PROT_READ | PROT_WRITE); + if (!mapped.isValid()) { + LOG(HAL, Error) << "Failed to mmap android blob buffer"; + continue; + } + + int jpeg_size = encoder->encode(buffer, mapped.maps()[0]); + if (jpeg_size < 0) { + LOG(HAL, Error) << "Failed to encode stream image"; + status = CAMERA3_BUFFER_STATUS_ERROR; + continue; + } + + /* + * Fill in the JPEG blob header. + * + * The mapped size of the buffer is being returned as + * substantially larger than the requested JPEG_MAX_SIZE + * (which is referenced from maxJpegBufferSize_). Utilise + * this static size to ensure the correct offset of the blob is + * determined. + * + * \todo Investigate if the buffer size mismatch is an issue or + * expected behaviour. + */ + uint8_t *resultPtr = mapped.maps()[0].data() + + maxJpegBufferSize_ - + sizeof(struct camera3_jpeg_blob); + auto *blob = reinterpret_cast(resultPtr); + blob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID; + blob->jpeg_size = jpeg_size; + + /* Update the JPEG result Metadata. */ + resultMetadata->addEntry(ANDROID_JPEG_SIZE, + &jpeg_size, 1); + + const uint32_t jpeg_quality = 95; + resultMetadata->addEntry(ANDROID_JPEG_QUALITY, + &jpeg_quality, 1); + + const uint32_t jpeg_orientation = 0; + resultMetadata->addEntry(ANDROID_JPEG_ORIENTATION, + &jpeg_orientation, 1); + } + + /* Prepare to call back the Android camera stack. */ camera3_capture_result_t captureResult = {}; captureResult.frame_number = descriptor->frameNumber; captureResult.num_output_buffers = descriptor->numBuffers; @@ -1334,10 +1546,10 @@ std::unique_ptr CameraDevice::getResultMetadata(int frame_number { /* * \todo Keep this in sync with the actual number of entries. - * Currently: 12 entries, 36 bytes + * Currently: 18 entries, 62 bytes */ std::unique_ptr resultMetadata = - std::make_unique(15, 50); + std::make_unique(18, 62); if (!resultMetadata->isValid()) { LOG(HAL, Error) << "Failed to allocate static metadata"; return nullptr; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index 4e5fb98c9a07..7be9e11977f7 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -23,15 +23,25 @@ #include "libcamera/internal/log.h" #include "libcamera/internal/message.h" +#include "jpeg/encoder.h" + class CameraMetadata; struct CameraStream { + CameraStream(libcamera::PixelFormat, libcamera::Size); + ~CameraStream(); + /* * The index of the libcamera StreamConfiguration as added during * configureStreams(). A single libcamera Stream may be used to deliver * one or more streams to the Android framework. */ unsigned int index; + + libcamera::PixelFormat format; + libcamera::Size size; + + Encoder *jpeg; }; class CameraDevice : protected libcamera::Loggable @@ -105,6 +115,8 @@ private: int facing_; int orientation_; + + unsigned int maxJpegBufferSize_; }; #endif /* __ANDROID_CAMERA_DEVICE_H__ */