From patchwork Fri Jun 5 14:09:55 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3945 Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 142C360410 for ; Fri, 5 Jun 2020 16:07:07 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id 7ABD62000D; Fri, 5 Jun 2020 14:07:06 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:09:55 +0200 Message-Id: <20200605141002.49119-2-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 1/8] android: camera_device: Initialize stream configuration X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:07 -0000 Initialize the stream configuration map by applying the Android Camera3 requested resolutions and formats to the libcamera Camera device. For each required format test a list of required and optional resolutions, construct a map to translate from Android format to the libcamera formats and store the available stream configuration to be provided to the Android framework through static metadata. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart --- src/android/camera_device.cpp | 224 ++++++++++++++++++++++++++++++++++ src/android/camera_device.h | 12 ++ 2 files changed, 236 insertions(+) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index b30263451b76..cb9e4a6bc15b 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,8 @@ #include "camera_device.h" #include "camera_ops.h" +#include + #include #include @@ -15,9 +17,75 @@ #include "libcamera/internal/utils.h" #include "camera_metadata.h" +#include "system/graphics.h" using namespace libcamera; +namespace { + +/* + * \var camera3Resolutions + * \brief The list of image resolutions defined as mandatory to be supported by + * the Android Camera3 specification + */ +const std::vector camera3Resolutions = { + { 320, 240 }, + { 640, 480 }, + { 1280, 720 }, + { 1920, 1080 } +}; + +/* + * \struct Camera3Format + * \brief Data associated with an Android format identifier + * \var libcameraFormats List of libcamera pixel formats compatible with the + * Android format + * \var scalerFormat The format identifier to be reported to the android + * framework through the static format configuration map + * \var formatName The human-readable representation of the Android format code + */ +struct Camera3Format { + std::vector libcameraFormats; + camera_metadata_enum_android_scaler_available_formats_t scalerFormat; + const char *formatName; +}; + +/* + * \var camera3FormatsMap + * \brief Associate Android format code with ancillary data + */ +const std::map camera3FormatsMap = { + { + HAL_PIXEL_FORMAT_BLOB, { + { PixelFormat(DRM_FORMAT_MJPEG) }, + ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, + "HAL_PIXEL_FORMAT_BLOB" + } + }, + + { + HAL_PIXEL_FORMAT_YCbCr_420_888, { + { PixelFormat(DRM_FORMAT_NV12), PixelFormat(DRM_FORMAT_NV21) }, + ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, + "HAL_PIXEL_FORMAT_YCbCr_420_888" + } + }, + + /* + * \todo Translate IMPLEMENTATION_DEFINED inspecting the + * gralloc usage flag. For now, copy the YCbCr_420 configuration. + */ + { + HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, { + { PixelFormat(DRM_FORMAT_NV12), PixelFormat(DRM_FORMAT_NV21) }, + ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, + "HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED" + } + }, +}; + +} /* namespace */ + LOG_DECLARE_CATEGORY(HAL); /* @@ -100,6 +168,162 @@ int CameraDevice::initialize() if (properties.contains(properties::Rotation)) orientation_ = properties.get(properties::Rotation); + int ret = camera_->acquire(); + if (ret) { + LOG(HAL, Error) << "Failed to temporarily acquire the camera"; + return ret; + } + + ret = initializeStreamConfigurations(); + camera_->release(); + return ret; +} + +/* + * Initialize the format conversion map to translate from Android format + * identifier to libcamera pixel formats and fill in the list of supported + * stream configurations to be reported to the Android camera framework through + * the static stream configuration metadata. + */ +int CameraDevice::initializeStreamConfigurations() +{ + /* + * Get the maximum output resolutions + * \todo Get this from the camera properties once defined + */ + std::unique_ptr cameraConfig = + camera_->generateConfiguration({ StillCapture }); + if (!cameraConfig) { + LOG(HAL, Error) << "Failed to get maximum resolution"; + return -EINVAL; + } + StreamConfiguration &cfg = cameraConfig->at(0); + + /* + * \todo JPEG - Adjust the maximum available resolution by taking the + * JPEG encoder requirements into account (alignment and aspect ratio). + */ + const Size maxRes = cfg.size; + LOG(HAL, Debug) << "Maximum supported resolution: " << maxRes.toString(); + + /* + * Build the list of supported image resolutions. + * + * The resolutions listed in camera3Resolution are mandatory to be + * supported, up to the camera maximum resolution. + * + * Augment the list by adding resolutions calculated from the camera + * maximum one. + */ + std::vector cameraResolutions; + std::copy_if(camera3Resolutions.begin(), camera3Resolutions.end(), + std::back_inserter(cameraResolutions), + [&](const Size &res) { return res < maxRes; }); + + /* Camera3 specification suggests to add 1/2 and 1/4 max resolution. */ + for (unsigned int divider = 2;; divider <<= 1) { + Size derivedSize{ + maxRes.width / divider, + maxRes.height / divider, + }; + + if (derivedSize.width < 320 || + derivedSize.height < 240) + break; + + cameraResolutions.push_back(derivedSize); + } + cameraResolutions.push_back(maxRes); + + /* Remove duplicated entries from the list of supported resolutions. */ + std::sort(cameraResolutions.begin(), cameraResolutions.end()); + auto last = std::unique(cameraResolutions.begin(), cameraResolutions.end()); + cameraResolutions.erase(last, cameraResolutions.end()); + + /* + * Build the list of supported camera formats. + * + * To each Android format a list of compatible libcamera formats is + * associated. The first libcamera format that tests successful is added + * to the format translation map used when configuring the streams. + * It is then tested against the list of supported camera resolutions to + * build the stream configuration map reported through the camera static + * metadata. + */ + for (const auto &format : camera3FormatsMap) { + int androidFormat = format.first; + const Camera3Format &camera3Format = format.second; + const std::vector &libcameraFormats = + camera3Format.libcameraFormats; + + /* + * Test the libcamera formats that can produce images + * compatible with the Android defined format. + */ + PixelFormat mappedFormat{}; + for (const PixelFormat &pixelFormat : libcameraFormats) { + /* \todo Fixed mapping for JPEG. */ + if (androidFormat == HAL_PIXEL_FORMAT_BLOB) { + mappedFormat = PixelFormat(DRM_FORMAT_MJPEG); + break; + } + + /* + * The stream configuration size can be adjusted, + * not the pixel format. + * + * \todo This could be simplified once all pipeline + * handlers will report the StreamFormats list of + * supported formats. + */ + cfg.pixelFormat = pixelFormat; + + CameraConfiguration::Status status = cameraConfig->validate(); + if (status != CameraConfiguration::Invalid && + cfg.pixelFormat == pixelFormat) { + mappedFormat = pixelFormat; + break; + } + } + if (!mappedFormat.isValid()) { + LOG(HAL, Error) << "Failed to map Android format " + << camera3Format.formatName << " (" + << utils::hex(androidFormat) << ")"; + return -EINVAL; + } + + /* + * Record the mapping and then proceed to generate the + * stream configurations map, by testing the image resolutions. + */ + formatsMap_[androidFormat] = mappedFormat; + + for (const Size &res : cameraResolutions) { + cfg.pixelFormat = mappedFormat; + cfg.size = res; + + CameraConfiguration::Status status = cameraConfig->validate(); + /* + * Unconditionally report we can produce JPEG. + * + * \todo The JPEG stream will be implemented as an + * HAL-only stream, but some cameras can produce it + * directly. As of now, claim support for JPEG without + * inspecting where the JPEG stream is produced. + */ + if (androidFormat != HAL_PIXEL_FORMAT_BLOB && + status != CameraConfiguration::Valid) + continue; + + streamConfigurations_.push_back({ res, camera3Format.scalerFormat }); + } + } + + LOG(HAL, Debug) << "Collected stream configuration map: "; + for (const auto &entry : streamConfigurations_) + LOG(HAL, Debug) << "{ " << entry.resolution.toString() << " - " + << utils::hex(entry.androidScalerCode) << " }"; + return 0; } diff --git a/src/android/camera_device.h b/src/android/camera_device.h index a87f7623c790..4e8911da5770 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -7,12 +7,15 @@ #ifndef __ANDROID_CAMERA_DEVICE_H__ #define __ANDROID_CAMERA_DEVICE_H__ +#include #include +#include #include #include #include +#include #include #include @@ -59,6 +62,12 @@ private: camera3_stream_buffer_t *buffers; }; + struct Camera3StreamConfiguration { + libcamera::Size resolution; + int androidScalerCode; + }; + + int initializeStreamConfigurations(); void notifyShutter(uint32_t frameNumber, uint64_t timestamp); void notifyError(uint32_t frameNumber, camera3_stream_t *stream); std::unique_ptr getResultMetadata(int frame_number, @@ -75,6 +84,9 @@ private: std::map requestTemplates_; const camera3_callback_ops_t *callbacks_; + std::vector streamConfigurations_; + std::map formatsMap_; + int facing_; int orientation_; }; From patchwork Fri Jun 5 14:09:56 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3946 Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id C70E661375 for ; Fri, 5 Jun 2020 16:07:07 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id 39CE72000D; Fri, 5 Jun 2020 14:07:07 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:09:56 +0200 Message-Id: <20200605141002.49119-3-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 2/8] android: camera_device: Calculate metadata size X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:08 -0000 As we move to have more and more dynamically generated static metadata entries, the size of the metadata buffer has to be calculated dynamically inspecting the information collected from the camera. Provide a method to perform metadata buffers size calculation and use it when generating camera static metadata. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart --- src/android/camera_device.cpp | 34 ++++++++++++++++++++++++++++------ src/android/camera_device.h | 2 ++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index cb9e4a6bc15b..c65295a32f3f 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -8,6 +8,7 @@ #include "camera_device.h" #include "camera_ops.h" +#include #include #include @@ -368,6 +369,29 @@ void CameraDevice::setCallbacks(const camera3_callback_ops_t *callbacks) callbacks_ = callbacks; } +std::tuple CameraDevice::calculateStaticMetadataSize() +{ + /* + * \todo Keep this in sync with the actual number of entries. + * Currently: 50 entries, 647 bytes of static metadata + */ + uint32_t numEntries = 50; + uint32_t byteSize = 647; + + /* + * Calculate space occupation in bytes for dynamically built metadata + * entries. + * + * Each stream configuration entry requires 52 bytes: + * 4 32bits integers for ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS + * 1 32bits integer for ANDROID_SCALER_AVAILABLE_FORMATS + * 4 64bits integers for ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS + */ + byteSize += streamConfigurations_.size() * 52; + + return { numEntries, byteSize }; +} + /* * Return static information for the camera. */ @@ -381,12 +405,10 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() * example application, but a real camera implementation will require * more. */ - - /* - * \todo Keep this in sync with the actual number of entries. - * Currently: 50 entries, 666 bytes - */ - staticMetadata_ = new CameraMetadata(50, 700); + uint32_t numEntries; + uint32_t byteSize; + std::tie(numEntries, byteSize) = calculateStaticMetadataSize(); + staticMetadata_ = new CameraMetadata(numEntries, byteSize); if (!staticMetadata_->isValid()) { LOG(HAL, Error) << "Failed to allocate static metadata"; delete staticMetadata_; diff --git a/src/android/camera_device.h b/src/android/camera_device.h index 4e8911da5770..ed11410a5577 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -68,6 +69,7 @@ private: }; int initializeStreamConfigurations(); + std::tuple calculateStaticMetadataSize(); void notifyShutter(uint32_t frameNumber, uint64_t timestamp); void notifyError(uint32_t frameNumber, camera3_stream_t *stream); std::unique_ptr getResultMetadata(int frame_number, From patchwork Fri Jun 5 14:09:57 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3947 Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CFDB461626 for ; Fri, 5 Jun 2020 16:07:09 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id B0BB22000D; Fri, 5 Jun 2020 14:07:07 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:09:57 +0200 Message-Id: <20200605141002.49119-4-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 3/8] android: camera_device: Replace hardcoded stream configuration X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:10 -0000 Replace the hardcoded stream configuration map with the information collected at CameraDevice initialization time. Reviewed-by: Laurent Pinchart Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 41 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index c65295a32f3f..7e3df2503468 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -655,23 +655,24 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM, &maxDigitalZoom, 1); - std::vector availableStreamFormats = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, - }; + std::vector availableStreamFormats; + availableStreamFormats.reserve(streamConfigurations_.size()); + std::transform(streamConfigurations_.begin(), streamConfigurations_.end(), + std::back_inserter(availableStreamFormats), + [](const auto &entry) { return entry.androidScalerCode; }); staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_FORMATS, availableStreamFormats.data(), availableStreamFormats.size()); - std::vector availableStreamConfigurations = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, 2560, 1920, - ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT, - }; + std::vector availableStreamConfigurations; + availableStreamConfigurations.reserve(streamConfigurations_.size() * 4); + for (const auto &entry : streamConfigurations_) { + availableStreamConfigurations.push_back(entry.androidScalerCode); + availableStreamConfigurations.push_back(entry.resolution.width); + availableStreamConfigurations.push_back(entry.resolution.height); + availableStreamConfigurations.push_back( + ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT); + } staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, availableStreamConfigurations.data(), availableStreamConfigurations.size()); @@ -683,11 +684,15 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() availableStallDurations.data(), availableStallDurations.size()); - std::vector minFrameDurations = { - ANDROID_SCALER_AVAILABLE_FORMATS_BLOB, 2560, 1920, 33333333, - ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED, 2560, 1920, 33333333, - ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888, 2560, 1920, 33333333, - }; + /* \todo Collect the minimum frame duration from the camera. */ + std::vector minFrameDurations; + minFrameDurations.reserve(streamConfigurations_.size() * 4); + for (const auto &entry : streamConfigurations_) { + minFrameDurations.push_back(entry.androidScalerCode); + minFrameDurations.push_back(entry.resolution.width); + minFrameDurations.push_back(entry.resolution.height); + minFrameDurations.push_back(33333333); + } staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS, minFrameDurations.data(), minFrameDurations.size()); From patchwork Fri Jun 5 14:09:58 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3948 Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7F7B4615F7 for ; Fri, 5 Jun 2020 16:07:10 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id E09E420002; Fri, 5 Jun 2020 14:07:09 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:09:58 +0200 Message-Id: <20200605141002.49119-5-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 4/8] android: camera_device: Translate Android format X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:10 -0000 Translate the Android format code to the libcamera format code at stream configuration time, using the translation map built at camera device initialization time. Reviewed-by: Laurent Pinchart Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 7e3df2503468..bbfc2464c908 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -939,12 +939,26 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) << ", format: " << utils::hex(stream->format); } - /* Hardcode viewfinder role, collecting sizes from the stream config. */ + /* Only one stream is supported. */ if (stream_list->num_streams != 1) { LOG(HAL, Error) << "Only one stream supported"; return -EINVAL; } + camera3_stream_t *camera3Stream = stream_list->streams[0]; + + /* Translate Android format code to libcamera pixel format. */ + auto it = formatsMap_.find(camera3Stream->format); + if (it == formatsMap_.end()) { + LOG(HAL, Error) << "Requested format " + << utils::hex(camera3Stream->format) + << " not supported"; + return -EINVAL; + } + /* + * Hardcode viewfinder role, replacing the generated configuration + * parameters with the ones requested by the Android framework. + */ StreamRoles roles = { StreamRole::Viewfinder }; config_ = camera_->generateConfiguration(roles); if (!config_ || config_->empty()) { @@ -952,17 +966,10 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) return -EINVAL; } - /* Only one stream is supported. */ - camera3_stream_t *camera3Stream = stream_list->streams[0]; StreamConfiguration *streamConfiguration = &config_->at(0); streamConfiguration->size.width = camera3Stream->width; streamConfiguration->size.height = camera3Stream->height; - - /* - * \todo We'll need to translate from Android defined pixel format codes - * to the libcamera image format codes. For now, do not change the - * format returned from Camera::generateConfiguration(). - */ + streamConfiguration->pixelFormat = it->second; switch (config_->validate()) { case CameraConfiguration::Valid: From patchwork Fri Jun 5 14:09:59 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3949 X-Patchwork-Delegate: jacopo@jmondi.org Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6E7CB61645 for ; Fri, 5 Jun 2020 16:07:11 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id A143120003; Fri, 5 Jun 2020 14:07:10 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:09:59 +0200 Message-Id: <20200605141002.49119-6-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 5/8] libcamera: properties: Add FrameDurationLimits X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:11 -0000 Add a camera property to express the minimum and maximum frame durations. Signed-off-by: Jacopo Mondi --- Cc Naush and Dave as this could potentially conflict with their on-going FPS handling series. Sending it out for discussion. --- src/libcamera/property_ids.yaml | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) -- 2.27.0 diff --git a/src/libcamera/property_ids.yaml b/src/libcamera/property_ids.yaml index ce627fa042ba..d703ab31eaac 100644 --- a/src/libcamera/property_ids.yaml +++ b/src/libcamera/property_ids.yaml @@ -386,4 +386,51 @@ controls: | | | | +--------------------+ + + - FrameDurationLimits: + type: int32_t + size: [2] + description: | + The camera supported frame durations interval. + + This property reports the camera minimum and maximum frame durations (in + this order) expressed in nanoseconds to report the limits of the + achievable frame rate. + + Camera devices should here report durations calculated by inspecting the + camera sensor supported frame rate interval, and adding to it any + known additional delay caused by the image acquisition process that + incurs in the time between a frame is captured and delivered to + applications. + + The here reported durations represent the time interval that occurs + between the delivery of two consecutive frames for the fastest and + slower streams, without considering additional delays introduced by the + sharing of system resources when multiple streams are captured at the + same time. + + This implies that the minimum reported frame duration, which corresponds + to the highest possible camera frame rate, is calculated without taking + into consideration how multiple streams capture requests sent to + the camera as part of the same capture session might influence the frame + delivery rate by, in example, introducing delays due to the requirement + of time-sharing components part of the image acquisition pipeline. In + example, request containing two scaled-down streams might require + time-sharing the single scaler available in the system, introducing an + additional delay that prevents the camera to deliver frames at the + here reported rate. + + In the same way, the maximum frame duration, which corresponds to the + lowest possible frame rate, does not take into consideration additional + processing delays introduced by image encoding and processing that + happens sporadically, in example, to produce still images in JPEG + format. + + As a consequence, application should inspect this property to know the + camera capability limits, but should not assume the here reported values + are achievable under all circumstances. + + # \todo Expand to better describe how stalling streams (ie JPEG) impacts + # the frame rate calculation. + ... From patchwork Fri Jun 5 14:10:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3950 X-Patchwork-Delegate: jacopo@jmondi.org Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CF3C160410 for ; Fri, 5 Jun 2020 16:07:12 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id 9102A20002; Fri, 5 Jun 2020 14:07:11 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:10:00 +0200 Message-Id: <20200605141002.49119-7-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 6/8] libcamera: camera_sensor: Break out properties initialization X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:13 -0000 Refactor the CameraSensor properties initialization to a dedicated function as it is expected to grow as we augment the number of properties. While at it, move documentation of the properties() method to match the declaration order in the class definition. Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham Reviewed-by: Niklas Söderlund Signed-off-by: Jacopo Mondi --- include/libcamera/internal/camera_sensor.h | 2 + src/libcamera/camera_sensor.cpp | 83 ++++++++++++---------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/include/libcamera/internal/camera_sensor.h b/include/libcamera/internal/camera_sensor.h index d79bd9ce9d58..44dd4b099913 100644 --- a/include/libcamera/internal/camera_sensor.h +++ b/include/libcamera/internal/camera_sensor.h @@ -69,6 +69,8 @@ protected: std::string logPrefix() const; private: + int initProperties(); + const MediaEntity *entity_; std::unique_ptr subdev_; unsigned int pad_; diff --git a/src/libcamera/camera_sensor.cpp b/src/libcamera/camera_sensor.cpp index b14b4051dca6..b9428175c55e 100644 --- a/src/libcamera/camera_sensor.cpp +++ b/src/libcamera/camera_sensor.cpp @@ -205,7 +205,47 @@ int CameraSensor::init() if (ret < 0) return ret; - /* Retrieve and store the camera sensor properties. */ + /* Enumerate, sort and cache media bus codes and sizes. */ + formats_ = subdev_->formats(pad_); + if (formats_.isEmpty()) { + LOG(CameraSensor, Error) << "No image format found"; + return -EINVAL; + } + + mbusCodes_ = formats_.formats(); + std::sort(mbusCodes_.begin(), mbusCodes_.end()); + + for (const auto &format : formats_.data()) { + const std::vector &ranges = format.second; + std::transform(ranges.begin(), ranges.end(), std::back_inserter(sizes_), + [](const SizeRange &range) { return range.max; }); + } + + std::sort(sizes_.begin(), sizes_.end()); + + /* Remove duplicates. */ + auto last = std::unique(sizes_.begin(), sizes_.end()); + sizes_.erase(last, sizes_.end()); + + /* + * The sizes_ vector is sorted in ascending order, the resolution is + * thus the last element of the vector. + */ + resolution_ = sizes_.back(); + + return initProperties(); +} + +/** + * \brief Initialize the camera sensor standard properties + * + * This method initializes the camera sensor standard properties, by inspecting + * the control information reported by the sensor subdevice. + * + * \return 0 on success, a negative error code otherwise + */ +int CameraSensor::initProperties() +{ const ControlInfoMap &controls = subdev_->controls(); int32_t propertyValue; @@ -243,35 +283,6 @@ int CameraSensor::init() propertyValue = 0; properties_.set(properties::Rotation, propertyValue); - /* Enumerate, sort and cache media bus codes and sizes. */ - formats_ = subdev_->formats(pad_); - if (formats_.isEmpty()) { - LOG(CameraSensor, Error) << "No image format found"; - return -EINVAL; - } - - mbusCodes_ = formats_.formats(); - std::sort(mbusCodes_.begin(), mbusCodes_.end()); - - for (const auto &format : formats_.data()) { - const std::vector &ranges = format.second; - std::transform(ranges.begin(), ranges.end(), std::back_inserter(sizes_), - [](const SizeRange &range) { return range.max; }); - } - - std::sort(sizes_.begin(), sizes_.end()); - - /* Remove duplicates. */ - auto last = std::unique(sizes_.begin(), sizes_.end()); - sizes_.erase(last, sizes_.end()); - - /* - * The sizes_ vector is sorted in ascending order, the resolution is - * thus the last element of the vector. - */ - resolution_ = sizes_.back(); - - return 0; } /** @@ -438,12 +449,6 @@ ControlList CameraSensor::getControls(const std::vector &ids) return subdev_->getControls(ids); } -/** - * \fn CameraSensor::properties() - * \brief Retrieve the camera sensor properties - * \return The list of camera sensor properties - */ - /** * \brief Write controls to the sensor * \param[in] ctrls The list of controls to write @@ -535,6 +540,12 @@ int CameraSensor::sensorInfo(CameraSensorInfo *info) const return 0; } +/** + * \fn CameraSensor::properties() + * \brief Retrieve the camera sensor properties + * \return The list of camera sensor properties + */ + std::string CameraSensor::logPrefix() const { return "'" + entity_->name() + "'"; From patchwork Fri Jun 5 14:10:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3951 X-Patchwork-Delegate: jacopo@jmondi.org Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 5920061645 for ; Fri, 5 Jun 2020 16:07:13 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id F180A20009; Fri, 5 Jun 2020 14:07:12 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:10:01 +0200 Message-Id: <20200605141002.49119-8-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 7/8] libcamera: camera_sensor: Initialize frame durations X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:13 -0000 Calculate the camera minimum and maximum frame durations, and register the corresponding FrameDurationLimits property with those values. Signed-off-by: Jacopo Mondi --- src/libcamera/camera_sensor.cpp | 74 +++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/libcamera/camera_sensor.cpp b/src/libcamera/camera_sensor.cpp index b9428175c55e..b144940f047d 100644 --- a/src/libcamera/camera_sensor.cpp +++ b/src/libcamera/camera_sensor.cpp @@ -283,6 +283,80 @@ int CameraSensor::initProperties() propertyValue = 0; properties_.set(properties::Rotation, propertyValue); + /* Frame durations. */ + std::map controlLimits; + std::array controlIds = { + V4L2_CID_VBLANK, + V4L2_CID_HBLANK, + V4L2_CID_PIXEL_RATE + }; + for (uint32_t controlId : controlIds) { + /* + * Make sure the subdevice reports VBLANK, HBLANK and PIXEL_RATE + * controls and collect the control information. + */ + auto it = controls.find(controlId); + if (it == controls.end()) { + LOG(CameraSensor, Error) + << "Camera sensor does not support control " + << controlId; + return -EINVAL; + } + + controlLimits[controlId] = &it->second; + } + + /* + * While the maximum line and frame sizes can be calculated by adding + * each control's maximum supported value to the current configuration, + * the minimum sizes shall be calculated by using the minimum available + * resolution. Get the current and smalles available size then calculate + * the frame durations. + */ + V4L2SubdeviceFormat fmt{}; + int ret = subdev_->getFormat(0, &fmt); + if (ret) + return ret; + const Size ¤tSize = fmt.size; + const Size &minSize = *sizes_.begin(); + + std::vector durations; + int32_t duration; + + /* + * Pixel rate is reported in Hz while frame duration is expressed in + * nanoseconds. Adjust it and calculate the minimum and maximum + * durations. + */ + int64_t pixelRate = controlLimits[V4L2_CID_PIXEL_RATE]->max().get(); + float rate = static_cast(pixelRate) / 1e9F; + + /* Minimum frame duration: higher frame rate. */ + duration = (minSize.width + + controlLimits[V4L2_CID_HBLANK]->min().get()) + * (minSize.height + + controlLimits[V4L2_CID_VBLANK]->min().get()); + duration = static_cast(static_cast(duration) / rate); + durations.push_back(duration); + + pixelRate = controlLimits[V4L2_CID_PIXEL_RATE]->min().get(); + rate = static_cast(pixelRate) / 1e9F; + + /* Maximum frame duration: lower frame rate. */ + duration = (currentSize.width + + controlLimits[V4L2_CID_HBLANK]->max().get()) + * (currentSize.height + + controlLimits[V4L2_CID_VBLANK]->max().get()); + duration = static_cast(static_cast(duration) / rate); + durations.push_back(duration); + + properties_.set(properties::FrameDurationLimits, durations); + + LOG(CameraSensor, Debug) << "Frame durations interval = [" + << durations[0] << " - " + << durations[1] << "]"; + + return 0; } /** From patchwork Fri Jun 5 14:10:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacopo Mondi X-Patchwork-Id: 3952 X-Patchwork-Delegate: jacopo@jmondi.org Return-Path: Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id DC7CD60410 for ; Fri, 5 Jun 2020 16:07:13 +0200 (CEST) X-Originating-IP: 93.34.118.233 Received: from localhost.localdomain (93-34-118-233.ip49.fastwebnet.it [93.34.118.233]) (Authenticated sender: jacopo@jmondi.org) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id 761E820006; Fri, 5 Jun 2020 14:07:13 +0000 (UTC) From: Jacopo Mondi To: libcamera-devel@lists.libcamera.org Date: Fri, 5 Jun 2020 16:10:02 +0200 Message-Id: <20200605141002.49119-9-jacopo@jmondi.org> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200605141002.49119-1-jacopo@jmondi.org> References: <20200605141002.49119-1-jacopo@jmondi.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 8/8] android: camera_device: Report frame durations X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Jun 2020 14:07:14 -0000 Report the minimum frame durations for each collected stream configuration by inspecting the camera reported FrameDurationLimits property. Signed-off-by: Jacopo Mondi --- src/android/camera_device.cpp | 33 +++++++++++++++++++++++++++++---- src/android/camera_device.h | 3 ++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index bbfc2464c908..fdcbada754f6 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -175,7 +175,7 @@ int CameraDevice::initialize() return ret; } - ret = initializeStreamConfigurations(); + ret = initializeStreamConfigurations(properties); camera_->release(); return ret; } @@ -186,8 +186,27 @@ int CameraDevice::initialize() * stream configurations to be reported to the Android camera framework through * the static stream configuration metadata. */ -int CameraDevice::initializeStreamConfigurations() +int CameraDevice::initializeStreamConfigurations(const ControlList &properties) { + /* + * Collect minimum frame durations from the camera. Default to it + * 33.3 msec if the camera does not report frame duration information. + * + * \todo The frame durations interval reported by the camera through the + * FrameDurationLimits property represents the camera minimum and + * maximum durations, regardless of the currently configured image + * format and resolution. As we are currently missing an API to report + * the frame durations for a specific configuration and Android requires + * the minimum frame duration to be reported per-configuration, as of + * now report the duration value for all supported stream + * configurations. + */ + int64_t minFrameDuration = 33333333; + if (properties.contains(properties::FrameDurationLimits)) { + Span durationLimits = properties.get(properties::FrameDurationLimits); + minFrameDuration = static_cast(durationLimits[0]); + } + /* * Get the maximum output resolutions * \todo Get this from the camera properties once defined @@ -316,7 +335,13 @@ int CameraDevice::initializeStreamConfigurations() status != CameraConfiguration::Valid) continue; - streamConfigurations_.push_back({ res, camera3Format.scalerFormat }); + /* + * Default the minimum frame duration and update it + * later. + */ + streamConfigurations_.push_back({ res, + camera3Format.scalerFormat, + minFrameDuration }); } } @@ -691,7 +716,7 @@ const camera_metadata_t *CameraDevice::getStaticMetadata() minFrameDurations.push_back(entry.androidScalerCode); minFrameDurations.push_back(entry.resolution.width); minFrameDurations.push_back(entry.resolution.height); - minFrameDurations.push_back(33333333); + minFrameDurations.push_back(entry.minFrameDuration); } staticMetadata_->addEntry(ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS, minFrameDurations.data(), diff --git a/src/android/camera_device.h b/src/android/camera_device.h index ed11410a5577..b689d67263b0 100644 --- a/src/android/camera_device.h +++ b/src/android/camera_device.h @@ -66,9 +66,10 @@ private: struct Camera3StreamConfiguration { libcamera::Size resolution; int androidScalerCode; + int64_t minFrameDuration; }; - int initializeStreamConfigurations(); + int initializeStreamConfigurations(const libcamera::ControlList &properties); std::tuple calculateStaticMetadataSize(); void notifyShutter(uint32_t frameNumber, uint64_t timestamp); void notifyError(uint32_t frameNumber, camera3_stream_t *stream);