From patchwork Thu Jun 4 13:35:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3915 X-Patchwork-Delegate: jacopo@jmondi.org Return-Path: Received: from relay5-d.mail.gandi.net (relay5-d.mail.gandi.net [217.70.183.197]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D6C5C610A5 for ; Thu, 4 Jun 2020 15:31:46 +0200 (CEST) X-Originating-IP: 2.224.242.101 Received: from uno.localdomain (2-224-242-101.ip172.fastwebnet.it [2.224.242.101]) (Authenticated sender: jacopo@jmondi.org) by relay5-d.mail.gandi.net (Postfix) with ESMTPSA id 59CFD1C000E; Thu, 4 Jun 2020 13:31:46 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Thu, 4 Jun 2020 15:35:00 +0200 Message-Id: <20200604133503.14689-2-jacopo@jmondi.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200604133503.14689-1-jacopo@jmondi.org> References: <20200604133503.14689-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 1/4] android: camera_device: Build stream configuration X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Jun 2020 13:31:47 -0000 Build the stream configuration map by applying the Android Camera3 requested resolutions and formats to the libcamera Camera device. For each required format test a list of required and optional resolutions, construct a map to translate from Android format to the libcamera formats and store the available stream configuration to be provided to the Android framework through static metadata. Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 218 ++++++++++++++++++++++++++++++++++ src/android/camera_device.h | 12 ++ 2 files changed, 230 insertions(+) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index b30263451b76..0c462f53c9cc 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,8 @@ #include "camera_device.h" #include "camera_ops.h" +#include + #include #include @@ -15,9 +17,70 @@ #include "libcamera/internal/utils.h" #include "camera_metadata.h" +#include "system/graphics.h" using namespace libcamera; +namespace { + +/* + * \var camera3Resolutions + * \brief The list of image resolutions defined as mandatory to be supported by + * the Android Camera3 specification + */ +const std::vector camera3Resolutions = { + { 320, 240 }, + { 640, 480 }, + { 1280, 720 }, + { 1920, 1080 } +}; + +/* + * \struct Camera3Format + * \brief Data associated with an Android format identifier + * \var Camera3Format::libcameraFormats: List of libcamera pixel formats + * compatible with the Android format + * \var Camera3Format::scalerFormat: The format identifier to be reported to the + * android framework through the static format configuration map + */ +struct Camera3Format { + std::vector libcameraFormats; + camera_metadata_enum_android_scaler_available_formats_t scalerFormat; +}; + +/* + * \var camera3FormatsMap + * \brief Associate Android format code with ancillary data + */ +const std::map camera3FormatsMap = { + { + HAL_PIXEL_FORMAT_BLOB, { + { PixelFormat(DRM_FORMAT_MJPEG) }, + ANDROID_SCALER_AVAILABLE_FORMATS_BLOB + } + }, + + { + HAL_PIXEL_FORMAT_YCbCr_420_888, { + { PixelFormat(DRM_FORMAT_NV12), PixelFormat(DRM_FORMAT_NV21) }, + ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888 + } + }, + + /* + * \todo Translate IMPLEMENTATION_DEFINED inspecting the + * gralloc usage flag. For now, copy the YCbCr_420 configuration. + */ + { + HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, { + { PixelFormat(DRM_FORMAT_NV12), PixelFormat(DRM_FORMAT_NV21) }, + ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888 + } + }, +}; + +} /* namespace */ + LOG_DECLARE_CATEGORY(HAL); /* @@ -100,6 +163,161 @@ int CameraDevice::initialize() if (properties.contains(properties::Rotation)) orientation_ = properties.get(properties::Rotation); + int ret = camera_->acquire(); + if (ret) { + LOG(HAL, Error) << "Failed to temporarily acquire the camera"; + return ret; + } + + ret = initializeFormats(); + camera_->release(); + return ret; +} + +/* + * Initialize the format conversion map to translate from Android format + * identifier to libcamera pixel formats and fill in the list of supported + * stream configurations to be reported to the Android camera framework through + * the static stream configuration metadata. + */ +int CameraDevice::initializeFormats() +{ + /* + * Get the maximum output resolutions + * \todo Get this from the camera properties once defined + */ + std::unique_ptr cameraConfig = + camera_->generateConfiguration({ StillCapture }); + if (!cameraConfig) { + LOG(HAL, Error) << "Failed to get maximum resolution"; + return -EINVAL; + } + StreamConfiguration &cfg = cameraConfig->at(0); + + /* + * \todo JPEG - Adjust the maximum available resolution by taking the + * JPEG encoder requirements into account (alignment and aspect ratio). + */ + const Size maxRes = cfg.size; + LOG(HAL, Debug) << "Maximum supported resolution: " << maxRes.toString(); + + /* + * Build the list of supported image resolutions. + * + * The resolutions listed in camera3Resolution are mandatory to be + * supported, up to the camera maximum resolution. + * + * Augment the list by adding resolutions calculated from the camera + * maximum one. + */ + std::vector cameraResolutions; + std::copy_if(camera3Resolutions.begin(), camera3Resolutions.end(), + std::back_inserter(cameraResolutions), + [&](const Size &res) { return res < maxRes; }); + + /* Camera3 specification suggests to add 1/2 and 1/4 max resolution. */ + for (unsigned int divider = 2;; divider <<= 1) { + Size derivedSize{ + maxRes.width / divider, + maxRes.height / divider, + }; + + if (derivedSize.width < 320 || + derivedSize.height < 240) + break; + + cameraResolutions.push_back(derivedSize); + } + cameraResolutions.push_back(maxRes); + + /* Remove duplicated entries from the list of supported resolutions. */ + std::sort(cameraResolutions.begin(), cameraResolutions.end()); + auto last = std::unique(cameraResolutions.begin(), cameraResolutions.end()); + cameraResolutions.erase(last, cameraResolutions.end()); + + /* + * Build the list of supported camera formats. + * + * To each Android format a list of compatible libcamera formats is + * associated. The first libcamera format that tests successful is added + * to the format translation map used when configuring the streams. + * It is then tested against the list of supported camera resolutions to + * build the stream configuration map reported through the camera static + * metadata. + */ + for (const auto &format : camera3FormatsMap) { + int androidFormat = format.first; + const Camera3Format &camera3Format = format.second; + const std::vector &libcameraFormats = + camera3Format.libcameraFormats; + + /* + * Test the libcamera formats that can produce images + * compatible with the Android defined format. + */ + PixelFormat mappedFormat{}; + for (const PixelFormat &format : libcameraFormats) { + /* \todo Fixed mapping for JPEG. */ + if (androidFormat == HAL_PIXEL_FORMAT_BLOB) { + mappedFormat = PixelFormat(DRM_FORMAT_MJPEG); + break; + } + + /* + * The stream configuration size can be adjusted, + * not the pixel format. + * + * \todo This could be simplified once all pipeline + * handlers will report the StreamFormats list of + * supported formats. + */ + cfg.pixelFormat = format; + + CameraConfiguration::Status status = cameraConfig->validate(); + if (status != CameraConfiguration::Invalid && + cfg.pixelFormat == format) { + mappedFormat = format; + break; + } + } + if (!mappedFormat.isValid()) { + LOG(HAL, Error) << "Failed to map Android format " + << utils::hex(androidFormat); + return -EINVAL; + } + + /* + * Record the mapping and then proceed to generate the + * stream configurations map, by testing the image resolutions. + */ + formatsMap_[androidFormat] = mappedFormat; + + for (const Size &res : cameraResolutions) { + cfg.pixelFormat = mappedFormat; + cfg.size = res; + + CameraConfiguration::Status status = cameraConfig->validate(); + /* + * Unconditionally report we can produce JPEG. + * + * \todo The JPEG stream will be implemented as an + * HAL-only stream, but some cameras can produce it + * directly. As of now, claim support for JPEG without + * inspecting where the JPEG stream is produced. + */ + if (androidFormat != HAL_PIXEL_FORMAT_BLOB && + status != CameraConfiguration::Valid) + continue; + + streamConfigurations_.push_back({ res, camera3Format.scalerFormat }); + } + } + + LOG(HAL, Debug) << "Collected stream configuration map: "; + for (const auto &entry : streamConfigurations_) + LOG(HAL, Debug) << "{ " << entry.resolution.toString() << " - " + << utils::hex(entry.androidScalerCode); + return 0; } diff --git a/src/android/camera_device.h b/src/android/camera_device.h index a87f7623c790..d31b7233e795 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -7,12 +7,15 @@ #ifndef __ANDROID_CAMERA_DEVICE_H__ #define __ANDROID_CAMERA_DEVICE_H__ +#include #include +#include #include #include #include +#include #include #include @@ -59,6 +62,12 @@ private: camera3_stream_buffer_t *buffers; }; + struct Camera3StreamConfiguration { + libcamera::Size resolution; + int androidScalerCode; + }; + + int initializeFormats(); void notifyShutter(uint32_t frameNumber, uint64_t timestamp); void notifyError(uint32_t frameNumber, camera3_stream_t *stream); std::unique_ptr getResultMetadata(int frame_number, @@ -75,6 +84,9 @@ private: std::map requestTemplates_; const camera3_callback_ops_t *callbacks_; + std::vector streamConfigurations_; + std::map formatsMap_; + int facing_; int orientation_; }; From patchwork Thu Jun 4 13:35:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3916 Return-Path: Received: from relay5-d.mail.gandi.net (relay5-d.mail.gandi.net [217.70.183.197]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 828F5611AB for ; Thu, 4 Jun 2020 15:31:47 +0200 (CEST) X-Originating-IP: 2.224.242.101 Received: from uno.localdomain (2-224-242-101.ip172.fastwebnet.it [2.224.242.101]) (Authenticated sender: jacopo@jmondi.org) by relay5-d.mail.gandi.net (Postfix) with ESMTPSA id 10E3E1C0005; Thu, 4 Jun 2020 13:31:46 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Thu, 4 Jun 2020 15:35:01 +0200 Message-Id: <20200604133503.14689-3-jacopo@jmondi.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200604133503.14689-1-jacopo@jmondi.org> References: <20200604133503.14689-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 2/4] android: camera_device: Calculate metadata size X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Jun 2020 13:31:48 -0000 As we move to have more and more dynamically generated static metadata entries, the size of the metadata buffer has to be calculated dynamically inspecting the information collected from the camera. Provide a method to perform metadata buffers size calculation and use it when generating camera static metadata. Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 34 ++++++++++++++++++++++++++++------ src/android/camera_device.h | 2 ++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 0c462f53c9cc..1f438f0e265d 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,7 @@ #include "camera_device.h" #include "camera_ops.h" +#include #include #include @@ -362,6 +363,29 @@ void CameraDevice::setCallbacks(const camera3_callback_ops_t *callbacks) callbacks_ = callbacks; } +std::tuple CameraDevice::calculateStaticMetadataSize() +{ + /* + * \todo Keep this in sync with the actual number of entries. + * Currently: 50 entries, 647 bytes of static metadata + */ + uint32_t numEntries = 50; + uint32_t byteSize = 647; + + /* + * Calculate space occupation in bytes for dynamically built metadata + * entries. + * + * Each stream configuration entry requires 52 bytes: + * 4 32bits integers for ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS + * 1 32bits integer for ANDROID_SCALER_AVAILABLE_FORMATS + * 4 64bits integers for ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS + */ + byteSize += streamConfigurations_.size() * 52; + + return { numEntries, byteSize }; +} + /* * Return static information for the camera. */ @@ -375,12 +399,10 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() * example application, but a real camera implementation will require * more. */ - - /* - * \todo Keep this in sync with the actual number of entries. - * Currently: 50 entries, 666 bytes - */ - staticMetadata_ = new CameraMetadata(50, 700); + uint32_t numEntries; + uint32_t byteSize; + std::tie(numEntries, byteSize) = calculateStaticMetadataSize(); + staticMetadata_ = new CameraMetadata(numEntries, byteSize); if (!staticMetadata_->isValid()) { LOG(HAL, Error) << "Failed to allocate static metadata"; delete staticMetadata_; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index d31b7233e795..18a5f77f9ce5 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -68,6 +69,7 @@ private: }; int initializeFormats(); + std::tuple calculateStaticMetadataSize(); void notifyShutter(uint32_t frameNumber, uint64_t timestamp); void notifyError(uint32_t frameNumber, camera3_stream_t *stream); std::unique_ptr getResultMetadata(int frame_number, From patchwork Thu Jun 4 13:35:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3917 Return-Path: Received: from relay5-d.mail.gandi.net (relay5-d.mail.gandi.net [217.70.183.197]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 5710F610A5 for ; Thu, 4 Jun 2020 15:31:48 +0200 (CEST) X-Originating-IP: 2.224.242.101 Received: from uno.localdomain (2-224-242-101.ip172.fastwebnet.it [2.224.242.101]) (Authenticated sender: jacopo@jmondi.org) by relay5-d.mail.gandi.net (Postfix) with ESMTPSA id AB6421C0005; Thu, 4 Jun 2020 13:31:47 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Thu, 4 Jun 2020 15:35:02 +0200 Message-Id: <20200604133503.14689-4-jacopo@jmondi.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200604133503.14689-1-jacopo@jmondi.org> References: <20200604133503.14689-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 3/4] android: camera_device: Replace hardcoded stream configuration X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Jun 2020 13:31:49 -0000 Replace the hardcoded stream configuration map with the information collected at CameraDevice initialization time. Reviewed-by: Laurent Pinchart Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 41 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 1f438f0e265d..d2570e91fc61 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -649,23 +649,24 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM, &maxDigitalZoom, 1); - std::vector availableStreamFormats = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, - }; + std::vector availableStreamFormats; + availableStreamFormats.reserve(streamConfigurations_.size()); + std::transform(streamConfigurations_.begin(), streamConfigurations_.end(), + std::back_inserter(availableStreamFormats), + [](const auto &entry) { return entry.androidScalerCode; }); staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_FORMATS, availableStreamFormats.data(), availableStreamFormats.size()); - std::vector availableStreamConfigurations = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - }; + std::vector availableStreamConfigurations; + availableStreamConfigurations.reserve(streamConfigurations_.size() * 4); + for (const auto &entry : streamConfigurations_) { + availableStreamConfigurations.push_back(entry.androidScalerCode); + availableStreamConfigurations.push_back(entry.resolution.width); + availableStreamConfigurations.push_back(entry.resolution.height); + availableStreamConfigurations.push_back( + ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT); + } staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, availableStreamConfigurations.data(), availableStreamConfigurations.size()); @@ -677,11 +678,15 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() availableStallDurations.data(), availableStallDurations.size()); - std::vector minFrameDurations = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, 2560, 1920, 33333333, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, 2560, 1920, 33333333, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, 2560, 1920, 33333333, - }; + /* \todo Collect the minimum frame duration from the camera. */ + std::vector minFrameDurations; + minFrameDurations.reserve(streamConfigurations_.size() * 4); + for (const auto &entry : streamConfigurations_) { + minFrameDurations.push_back(entry.androidScalerCode); + minFrameDurations.push_back(entry.resolution.width); + minFrameDurations.push_back(entry.resolution.height); + minFrameDurations.push_back(33333333); + } staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS, minFrameDurations.data(), minFrameDurations.size()); From patchwork Thu Jun 4 13:35:03 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3918 Return-Path: Received: from relay5-d.mail.gandi.net (relay5-d.mail.gandi.net [217.70.183.197]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1DAC961027 for ; Thu, 4 Jun 2020 15:31:49 +0200 (CEST) X-Originating-IP: 2.224.242.101 Received: from uno.localdomain (2-224-242-101.ip172.fastwebnet.it [2.224.242.101]) (Authenticated sender: jacopo@jmondi.org) by relay5-d.mail.gandi.net (Postfix) with ESMTPSA id 7C0BA1C0016; Thu, 4 Jun 2020 13:31:48 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Thu, 4 Jun 2020 15:35:03 +0200 Message-Id: <20200604133503.14689-5-jacopo@jmondi.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200604133503.14689-1-jacopo@jmondi.org> References: <20200604133503.14689-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 4/4] android: camera_device: Translate Android format X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Jun 2020 13:31:49 -0000 Translate the Android format code to the libcamera format code at stream configuration time, using the translation map built at camera device initialization time. Reviewed-by: Laurent Pinchart Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index d2570e91fc61..915d9777fd78 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -933,12 +933,26 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) << ", format: " << utils::hex(stream->format); } - /* Hardcode viewfinder role, collecting sizes from the stream config. */ + /* Only one stream is supported. */ if (stream_list->num_streams != 1) { LOG(HAL, Error) << "Only one stream supported"; return -EINVAL; } + camera3_stream_t *camera3Stream = stream_list->streams[0]; + + /* Translate Android format code to libcamera pixel format. */ + auto it = formatsMap_.find(camera3Stream->format); + if (it == formatsMap_.end()) { + LOG(HAL, Error) << "Requested format " + << utils::hex(camera3Stream->format) + << " not supported"; + return -EINVAL; + } + /* + * Hardcode viewfinder role, replacing the generated configuration + * parameters with the ones requested by the Android framework. + */ StreamRoles roles = { StreamRole::Viewfinder }; config_ = camera_->generateConfiguration(roles); if (!config_ || config_->empty()) { @@ -946,17 +960,10 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) return -EINVAL; } - /* Only one stream is supported. */ - camera3_stream_t *camera3Stream = stream_list->streams[0]; StreamConfiguration *streamConfiguration = &config_->at(0); streamConfiguration->size.width = camera3Stream->width; streamConfiguration->size.height = camera3Stream->height; - - /* - * \todo We'll need to translate from Android defined pixel format codes - * to the libcamera image format codes. For now, do not change the - * format returned from Camera::generateConfiguration(). - */ + streamConfiguration->pixelFormat = it->second; switch (config_->validate()) { case CameraConfiguration::Valid: