From patchwork Tue Dec 1 04:25:14 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Hirokazu Honda X-Patchwork-Id: 10524 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id A7273BE177 for ; Tue, 1 Dec 2020 04:25:27 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 30B18634A1; Tue, 1 Dec 2020 05:25:27 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=chromium.org header.i=@chromium.org header.b="fmxZ0Cgt"; dkim-atps=neutral Received: from mail-pj1-x1041.google.com (mail-pj1-x1041.google.com [IPv6:2607:f8b0:4864:20::1041]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4637560329 for ; Tue, 1 Dec 2020 05:25:25 +0100 (CET) Received: by mail-pj1-x1041.google.com with SMTP id j13so443853pjz.3 for ; Mon, 30 Nov 2020 20:25:25 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org; s=google; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=S1e27upzQ1DCbTB8Ea+Ojf/hEaghpTdu0Po7DCAbPTw=; b=fmxZ0Cgtzwf5SHDpyLwL3Ze5zeH8+Zqpm/JWxN/p4lM/KyXIA0rKQ4szXP3MjNEXq8 Gjboz8iBAfSNIkcluPw2FnTqGxTNc2g1VL0G9ca6teNvb818i1NcvB/FB8JozSVF5HiL 6ELokBwlx0ViiTKs1dNVGoxhLUmYPZfoxJ4o4= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=S1e27upzQ1DCbTB8Ea+Ojf/hEaghpTdu0Po7DCAbPTw=; b=OPptJ5ZWpHhISaaI3d17YKKGeIMlHql2OT4z1SzRCFm90QSv10Ya6OoWjsdJJbbCVD zZeNlf5VJMDggzpXyg8+zy2A3vGRrWO+BToYX1n54z+lrEL6H1ji2Q5Xvg8tOdc/pVtR pnKfDxzlopDsQHzSFCrPLByha5O+BuEcffz74PTk4wF8JLSGaB5woswD+YpII6fGKCaF qPrTU+ssAy5c1FrM8ytjd5dSqoc5WXpb6W/D4CqMgI17jC7dG0zRAdqoxMRDvtf7Z0wN nHb+vH91XFi4ml4vRlnXgb1lD1PI0TQoF0Ni+0YpE4l7GW1k1ZY+b75pVc4CJ8CkCWtL 2Zvg== X-Gm-Message-State: AOAM5337EpIA6F6HlO32Bwjf4M2H7i0YFs2PgLpjqPVNHSD1i6N+G21M 9lQOHzCA7I3GDEvvG0SPixn0DOqLIcI3Sw== X-Google-Smtp-Source: ABdhPJyBMAtncniBRqpB0DeFJ5mtpul6kugbzf0q1VZqujkO0Z8yOcwJHS2PwhgmTP4KGqDcwg7XaQ== X-Received: by 2002:a17:902:9a4c:b029:d6:1f21:8021 with SMTP id x12-20020a1709029a4cb02900d61f218021mr1102429plv.58.1606796723168; Mon, 30 Nov 2020 20:25:23 -0800 (PST) Received: from hiroh.tok.corp.google.com ([2401:fa00:8f:2:de4a:3eff:fe7d:f78f]) by smtp.gmail.com with ESMTPSA id l17sm545619pjy.29.2020.11.30.20.25.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 30 Nov 2020 20:25:22 -0800 (PST) From: Hirokazu Honda To: libcamera-devel@lists.libcamera.org Date: Tue, 1 Dec 2020 13:25:14 +0900 Message-Id: <20201201042514.3209146-1-hiroh@chromium.org> X-Mailer: git-send-email 2.29.2.454.gaff20da3a2-goog MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v2] android: camera_device: Reorder configurations before request X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" This reorders configurations before calling CameraConfiguration::validate() so that the streams requested by an android HAL client can be achieved more likely. Signed-off-by: Hirokazu Honda --- src/android/camera_device.cpp | 158 +++++++++++++++++++++++++++++----- 1 file changed, 137 insertions(+), 21 deletions(-) -- 2.29.2.454.gaff20da3a2-goog diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index 4690346e..3de5a9f1 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -9,9 +9,11 @@ #include "camera_ops.h" #include "post_processor.h" +#include #include #include #include +#include #include #include @@ -128,6 +130,107 @@ const std::map camera3FormatsMap = { }, }; +struct Camera3StreamsToConfig { + std::vector streams; // Destination(s). + std::vector types; // CameraStream::Type(s). + StreamConfiguration config; // StreamConfiguration requested to a native camera. +}; + +/* + * Reorder the configurations so that CameraDevice can accept them as much as + * possible. + */ +std::vector createdSortedCamera3StreamsToConfigs( + std::vector unsortedStreamsToConfigs, + const camera3_stream_t *jpegStream) { + const size_t unsortedStreamsToConfigsSize = unsortedStreamsToConfigs.size(); + std::optional streamsToConfigForJpeg = std::nullopt; + if (jpegStream) { + for (auto it = unsortedStreamsToConfigs.begin(); + it != unsortedStreamsToConfigs.end(); it++) { + const auto& streams = it->streams; + if (std::find(streams.begin(), streams.end(), + jpegStream) != streams.end()) { + streamsToConfigForJpeg = *it; + unsortedStreamsToConfigs.erase(it); + break; + } + } + assert(streamsToConfigForJpeg.has_value()); + } + + std::map> formatToStreamsToConfigs; + for (const auto &streamsToConfig : unsortedStreamsToConfigs) { + const StreamConfiguration &config = streamsToConfig.config; + formatToStreamsToConfigs[config.pixelFormat].push_back(streamsToConfig); + + } + for (auto& [format, streamsToConfigs] : formatToStreamsToConfigs) { + /* Sorted by resolution. Smaller is put first. */ + std::sort(streamsToConfigs.begin(), streamsToConfigs.end(), + [](const auto &streamsToConfigA, const auto &streamsToConfigB) { + const Size &sizeA = streamsToConfigA.config.size; + const Size &sizeB = streamsToConfigA.config.size; + if (sizeA.width != sizeB.width) + return sizeA.width < sizeB.width; + return sizeA.height < sizeB.height; + }); + } + + std::vector sortedStreamsToConfigs; + /* + * NV12 is the most prioritized format. Put the configuration with NV12 + * and the largest resolution first. + */ + if (formatToStreamsToConfigs.find(formats::NV12) != formatToStreamsToConfigs.end()) { + auto& nv12StreamsToConfigs = formatToStreamsToConfigs[formats::NV12]; + const Size& nv12LargestSize = nv12StreamsToConfigs.back().config.size; + if (streamsToConfigForJpeg && + streamsToConfigForJpeg->config.pixelFormat == formats::NV12) { + const Size& nv12SizeForJpeg = streamsToConfigForJpeg->config.size; + /* + * If JPEG will be created from NV12 and the size is + * larger than the largest NV12 remained configurations, + * then put the NV12 configuration for JPEG first. + */ + if (nv12LargestSize.width < nv12SizeForJpeg.width && + nv12LargestSize.height < nv12SizeForJpeg.height) { + sortedStreamsToConfigs.push_back(*streamsToConfigForJpeg); + streamsToConfigForJpeg = std::nullopt; + } + } + sortedStreamsToConfigs.push_back(nv12StreamsToConfigs.back()); + nv12StreamsToConfigs.pop_back(); + } + + /* + * If the configuration for JPEG is there, then put it. + */ + if (streamsToConfigForJpeg) { + sortedStreamsToConfigs.push_back(*streamsToConfigForJpeg); + streamsToConfigForJpeg = std::nullopt; + } + + /* + * Put configurations with different formats and larger resolutions + * earlier. + */ + while (!formatToStreamsToConfigs.empty()) { + for (auto it = formatToStreamsToConfigs.begin(); + it != formatToStreamsToConfigs.end();) { + auto& streamsToConfigs = it->second; + if (streamsToConfigs.empty()) { + it = formatToStreamsToConfigs.erase(it); + continue; + } + sortedStreamsToConfigs.push_back(streamsToConfigs.back()); + streamsToConfigs.pop_back(); + } + } + assert(sortedStreamsToConfigs.size() == unsortedStreamsToConfigsSize); + return sortedStreamsToConfigs; +} + } /* namespace */ LOG_DECLARE_CATEGORY(HAL) @@ -1225,6 +1328,9 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) streams_.clear(); streams_.reserve(stream_list->num_streams); + std::vector streamsToConfigs; + streamsToConfigs.reserve(stream_list->num_streams); + /* First handle all non-MJPEG streams. */ camera3_stream_t *jpegStream = nullptr; for (unsigned int i = 0; i < stream_list->num_streams; ++i) { @@ -1255,14 +1361,12 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) continue; } - StreamConfiguration streamConfiguration; - streamConfiguration.size = size; - streamConfiguration.pixelFormat = format; - - config_->addConfiguration(streamConfiguration); - streams_.emplace_back(this, CameraStream::Type::Direct, - stream, config_->size() - 1); - stream->priv = static_cast(&streams_.back()); + Camera3StreamsToConfig streamsToConfig; + streamsToConfig.streams = {stream}; + streamsToConfig.types = {CameraStream::Type::Direct}; + streamsToConfig.config.size = size; + streamsToConfig.config.pixelFormat = format; + streamsToConfigs.push_back(std::move(streamsToConfig)); } /* Now handle the MJPEG streams, adding a new stream if required. */ @@ -1271,9 +1375,8 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) int index = -1; /* Search for a compatible stream in the non-JPEG ones. */ - for (unsigned int i = 0; i < config_->size(); i++) { - StreamConfiguration &cfg = config_->at(i); - + for (size_t i = 0; i < streamsToConfigs.size(); ++i) { + const auto& cfg = streamsToConfigs[i].config; /* * \todo The PixelFormat must also be compatible with * the encoder. @@ -1295,28 +1398,41 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list) * introduce a new stream to satisfy the request requirements. */ if (index < 0) { - StreamConfiguration streamConfiguration; - + Camera3StreamsToConfig streamsToConfig; /* * \todo The pixelFormat should be a 'best-fit' choice * and may require a validation cycle. This is not yet * handled, and should be considered as part of any * stream configuration reworks. */ - streamConfiguration.size.width = jpegStream->width; - streamConfiguration.size.height = jpegStream->height; - streamConfiguration.pixelFormat = formats::NV12; + streamsToConfig.config.size.width = jpegStream->width; + streamsToConfig.config.size.height = jpegStream->height; + streamsToConfig.config.pixelFormat = formats::NV12; + streamsToConfigs.push_back(std::move(streamsToConfig)); - LOG(HAL, Info) << "Adding " << streamConfiguration.toString() + LOG(HAL, Info) << "Adding " << streamsToConfig.config.toString() << " for MJPEG support"; type = CameraStream::Type::Internal; - config_->addConfiguration(streamConfiguration); - index = config_->size() - 1; + index = streamsToConfigs.size() - 1; } - streams_.emplace_back(this, type, jpegStream, index); - jpegStream->priv = static_cast(&streams_.back()); + streamsToConfigs[index].streams.push_back(jpegStream); + streamsToConfigs[index].types.push_back(type); + } + + streamsToConfigs = + createdSortedCamera3StreamsToConfigs(std::move(streamsToConfigs), + jpegStream); + for (auto& streamsToConfig : streamsToConfigs) { + config_->addConfiguration(streamsToConfig.config); + for (size_t i = 0; i < streamsToConfig.streams.size(); ++i) { + auto* stream = streamsToConfig.streams[i]; + const CameraStream::Type type = streamsToConfig.types[i]; + streams_.emplace_back(this, type, + stream, config_->size() - 1); + stream->priv = static_cast(&streams_.back()); + } } switch (config_->validate()) {