[libcamera-devel,v3,3/3] android: camera_device: Reorder configurations before requesting
diff mbox series

Message ID 20201207084218.2307410-3-hiroh@chromium.org
State Superseded
Headers show
Series
  • [libcamera-devel,v3,1/3] android: camera_device: Introduce Camera3StreamConfig
Related show

Commit Message

Hirokazu Honda Dec. 7, 2020, 8:42 a.m. UTC 
This reorders Camera3Configs before executing
CameraConfiguration::validate() to make it easier for the Camera
to satisfy the Android framework request.
---
 src/android/camera_device.cpp | 104 +++++++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 2 deletions(-)

--
2.29.2.576.ga3fc446d84-goog

Comments

Jacopo Mondi Dec. 7, 2020, 11:02 a.m. UTC | #1 
Hi Hiro,

On Mon, Dec 07, 2020 at 08:42:18AM +0000, Hirokazu Honda wrote:
> This reorders Camera3Configs before executing
> CameraConfiguration::validate() to make it easier for the Camera
> to satisfy the Android framework request.
> ---
>  src/android/camera_device.cpp | 104 +++++++++++++++++++++++++++++++++-
>  1 file changed, 102 insertions(+), 2 deletions(-)
>
> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
> index b7bf3d88..dcba4e7d 100644
> --- a/src/android/camera_device.cpp
> +++ b/src/android/camera_device.cpp
> @@ -9,6 +9,7 @@
>  #include "camera_ops.h"
>  #include "post_processor.h"
>
> +#include <optional>
>  #include <sys/mman.h>
>  #include <tuple>
>  #include <vector>
> @@ -27,6 +28,8 @@
>
>  using namespace libcamera;
>
> +LOG_DECLARE_CATEGORY(HAL)
> +
>  namespace {
>
>  /*
> @@ -140,9 +143,104 @@ struct Camera3StreamConfig {
>  	std::vector<CameraStream::Type> types;
>  	StreamConfiguration config;
>  };
> -} /* namespace */
>
> -LOG_DECLARE_CATEGORY(HAL)
> +/*
> + * Reorder the configurations so that CameraDevice can accept them as much as

s/CameraDevice/libcamera::Camera/

> + * possible. The sort rule is as follows.
> + * 1.) The configuration for NV12 request whose resolution is the largest.
> + * 2.) The configuration for JPEG request.
> + * 3.) Others. Larger resolutions and different formats are put earlier.
> + */
> +std::vector<Camera3StreamConfig> sortCamera3StreamConfigs(
> +	std::vector<Camera3StreamConfig> unsortedStreamConfigs,

just 'unsortedConfigs' ?

> +	const camera3_stream_t *jpegStream) {
> +	const size_t unsortedStreamConfigsSize = unsortedStreamConfigs.size();
> +	std::optional<Camera3StreamConfig> streamConfigForJpeg = std::nullopt;

These can be just 'unsortedSize' and 'jpegConfig'. Up to you.

Empty line ?
> +	if (jpegStream) {
> +		for (auto it = unsortedStreamConfigs.begin();
> +		     it != unsortedStreamConfigs.end(); it++) {
> +			const auto &streams = it->streams;
> +			if (std::find(streams.begin(), streams.end(),
> +				      jpegStream) != streams.end()) {
> +				streamConfigForJpeg = *it;
> +				unsortedStreamConfigs.erase(it);
> +				break;
> +			}
> +		}
> +                if (!streamConfigForJpeg)
> +                  LOG(HAL, Fatal) << "No Camera3StreamConfig is found for Jpeg";

What happened to indentation ?
Also, can this really happen ?

> +	}
> +
> +	std::map<uint32_t, std::vector<Camera3StreamConfig>> formatToConfigs;
> +	for (const auto &streamConfig : unsortedStreamConfigs) {
> +		const StreamConfiguration &config = streamConfig.config;
> +		formatToConfigs[config.pixelFormat].push_back(streamConfig);
> +
> +	}
> +	for (auto& [format, streamConfigs] : formatToConfigs) {

auto &[] or does the compiler complains ?

> +		/* Sorted by resolution. Smaller is put first. */
> +		std::sort(streamConfigs.begin(), streamConfigs.end(),
> +			  [](const auto &streamConfigA, const auto &streamConfigB) {
> +				  const Size &sizeA = streamConfigA.config.size;
> +				  const Size &sizeB = streamConfigB.config.size;
> +                                  return sizeA < sizeB;
> +			  });
> +	}
> +
> +	std::vector<Camera3StreamConfig> sortedStreamConfigs;
> +        sortedStreamConfigs.reserve(unsortedStreamConfigsSize);

Indentation ?

> +	/*
> +	 * NV12 is the most prioritized format. Put the configuration with NV12
> +	 * and the largest resolution first.
> +	 */
> +	const auto nv12Stream = formatToConfigs.find(formats::NV12);
> +	if (nv12Stream != formatToConfigs.end()) {
> +		auto &nv12StreamConfigs = nv12Stream->second;
> +		const Size &nv12LargestSize = nv12StreamConfigs.back().config.size;

Empty line ?

> +		/*
> +		 * If JPEG will be created from NV12 and the size is larger than
> +		 * the largest NV12 configurations, then put the NV12
> +		 * configuration for JPEG first.
> +		 */
> +		if (streamConfigForJpeg &&
> +		    streamConfigForJpeg->config.pixelFormat == formats::NV12) {
> +			const Size &nv12SizeForJpeg = streamConfigForJpeg->config.size;
> +			if (nv12LargestSize < nv12SizeForJpeg) {
> +				sortedStreamConfigs.push_back(*streamConfigForJpeg);
> +				streamConfigForJpeg = std::nullopt;
> +			}
> +		}
> +		sortedStreamConfigs.push_back(nv12StreamConfigs.back());
> +		nv12StreamConfigs.pop_back();
> +	}
> +
> +	/* If the configuration for JPEG is there, then put it. */
> +	if (streamConfigForJpeg) {
> +		sortedStreamConfigs.push_back(*streamConfigForJpeg);
> +		streamConfigForJpeg = std::nullopt;
> +	}
> +
> +	/*
> +	 * Put configurations with different formats and larger resolutions
> +	 * earlier.
> +	 */
> +	while (!formatToConfigs.empty()) {
> +		for (auto it = formatToConfigs.begin(); it != formatToConfigs.end();) {
> +			auto& streamConfigs = it->second;
> +			if (streamConfigs.empty()) {
> +				it = formatToConfigs.erase(it);
> +				continue;
> +			}
> +			sortedStreamConfigs.push_back(streamConfigs.back());
> +			streamConfigs.pop_back();
> +			it++;

AH! this it++ was not here in the previous version :)
Now I see what you wanted.

This makes me think that a LOG(Debug) printout of the the sorted
stream configs would help making sure we obtain what we intend.


> +		}
> +	}
> +	assert(sortedStreamConfigs.size() == unsortedStreamConfigsSize);
> +
> +	return sortedStreamConfigs;
> +}
> +} /* namespace */
>
>  MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer,
>  					 int flags)
> @@ -1333,6 +1431,8 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)
>  		streamConfigs[index].types.push_back(type);
>  	}
>
> +	streamConfigs = sortCamera3StreamConfigs(std::move(streamConfigs),
> +						  jpegStream);
>  	for (const auto &streamConfig : streamConfigs) {
>  		config_->addConfiguration(streamConfig.config);
>  		for (size_t i = 0; i < streamConfig.streams.size(); ++i) {
> --
> 2.29.2.576.ga3fc446d84-goog
Hirokazu Honda Dec. 8, 2020, 3:41 a.m. UTC | #2 
Hi Jacopo,

On Mon, Dec 7, 2020 at 8:02 PM Jacopo Mondi <jacopo@jmondi.org> wrote:
>
> Hi Hiro,
>
> On Mon, Dec 07, 2020 at 08:42:18AM +0000, Hirokazu Honda wrote:
> > This reorders Camera3Configs before executing
> > CameraConfiguration::validate() to make it easier for the Camera
> > to satisfy the Android framework request.
> > ---
> >  src/android/camera_device.cpp | 104 +++++++++++++++++++++++++++++++++-
> >  1 file changed, 102 insertions(+), 2 deletions(-)
> >
> > diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
> > index b7bf3d88..dcba4e7d 100644
> > --- a/src/android/camera_device.cpp
> > +++ b/src/android/camera_device.cpp
> > @@ -9,6 +9,7 @@
> >  #include "camera_ops.h"
> >  #include "post_processor.h"
> >
> > +#include <optional>
> >  #include <sys/mman.h>
> >  #include <tuple>
> >  #include <vector>
> > @@ -27,6 +28,8 @@
> >
> >  using namespace libcamera;
> >
> > +LOG_DECLARE_CATEGORY(HAL)
> > +
> >  namespace {
> >
> >  /*
> > @@ -140,9 +143,104 @@ struct Camera3StreamConfig {
> >       std::vector<CameraStream::Type> types;
> >       StreamConfiguration config;
> >  };
> > -} /* namespace */
> >
> > -LOG_DECLARE_CATEGORY(HAL)
> > +/*
> > + * Reorder the configurations so that CameraDevice can accept them as much as
>
> s/CameraDevice/libcamera::Camera/
>
> > + * possible. The sort rule is as follows.
> > + * 1.) The configuration for NV12 request whose resolution is the largest.
> > + * 2.) The configuration for JPEG request.
> > + * 3.) Others. Larger resolutions and different formats are put earlier.
> > + */
> > +std::vector<Camera3StreamConfig> sortCamera3StreamConfigs(
> > +     std::vector<Camera3StreamConfig> unsortedStreamConfigs,
>
> just 'unsortedConfigs' ?
>
> > +     const camera3_stream_t *jpegStream) {
> > +     const size_t unsortedStreamConfigsSize = unsortedStreamConfigs.size();
> > +     std::optional<Camera3StreamConfig> streamConfigForJpeg = std::nullopt;
>
> These can be just 'unsortedSize' and 'jpegConfig'. Up to you.
>
> Empty line ?
> > +     if (jpegStream) {
> > +             for (auto it = unsortedStreamConfigs.begin();
> > +                  it != unsortedStreamConfigs.end(); it++) {
> > +                     const auto &streams = it->streams;
> > +                     if (std::find(streams.begin(), streams.end(),
> > +                                   jpegStream) != streams.end()) {
> > +                             streamConfigForJpeg = *it;
> > +                             unsortedStreamConfigs.erase(it);
> > +                             break;
> > +                     }
> > +             }
> > +                if (!streamConfigForJpeg)
> > +                  LOG(HAL, Fatal) << "No Camera3StreamConfig is found for Jpeg";
>
> What happened to indentation ?
> Also, can this really happen ?
>

This must not happen. I originally assert this.

> > +     }
> > +
> > +     std::map<uint32_t, std::vector<Camera3StreamConfig>> formatToConfigs;
> > +     for (const auto &streamConfig : unsortedStreamConfigs) {
> > +             const StreamConfiguration &config = streamConfig.config;
> > +             formatToConfigs[config.pixelFormat].push_back(streamConfig);
> > +
> > +     }
> > +     for (auto& [format, streamConfigs] : formatToConfigs) {
>
> auto &[] or does the compiler complains ?
>
> > +             /* Sorted by resolution. Smaller is put first. */
> > +             std::sort(streamConfigs.begin(), streamConfigs.end(),
> > +                       [](const auto &streamConfigA, const auto &streamConfigB) {
> > +                               const Size &sizeA = streamConfigA.config.size;
> > +                               const Size &sizeB = streamConfigB.config.size;
> > +                                  return sizeA < sizeB;
> > +                       });
> > +     }
> > +
> > +     std::vector<Camera3StreamConfig> sortedStreamConfigs;
> > +        sortedStreamConfigs.reserve(unsortedStreamConfigsSize);
>
> Indentation ?
>
> > +     /*
> > +      * NV12 is the most prioritized format. Put the configuration with NV12
> > +      * and the largest resolution first.
> > +      */
> > +     const auto nv12Stream = formatToConfigs.find(formats::NV12);
> > +     if (nv12Stream != formatToConfigs.end()) {
> > +             auto &nv12StreamConfigs = nv12Stream->second;
> > +             const Size &nv12LargestSize = nv12StreamConfigs.back().config.size;
>
> Empty line ?
>
> > +             /*
> > +              * If JPEG will be created from NV12 and the size is larger than
> > +              * the largest NV12 configurations, then put the NV12
> > +              * configuration for JPEG first.
> > +              */
> > +             if (streamConfigForJpeg &&
> > +                 streamConfigForJpeg->config.pixelFormat == formats::NV12) {
> > +                     const Size &nv12SizeForJpeg = streamConfigForJpeg->config.size;
> > +                     if (nv12LargestSize < nv12SizeForJpeg) {
> > +                             sortedStreamConfigs.push_back(*streamConfigForJpeg);
> > +                             streamConfigForJpeg = std::nullopt;
> > +                     }
> > +             }
> > +             sortedStreamConfigs.push_back(nv12StreamConfigs.back());
> > +             nv12StreamConfigs.pop_back();
> > +     }
> > +
> > +     /* If the configuration for JPEG is there, then put it. */
> > +     if (streamConfigForJpeg) {
> > +             sortedStreamConfigs.push_back(*streamConfigForJpeg);
> > +             streamConfigForJpeg = std::nullopt;
> > +     }
> > +
> > +     /*
> > +      * Put configurations with different formats and larger resolutions
> > +      * earlier.
> > +      */
> > +     while (!formatToConfigs.empty()) {
> > +             for (auto it = formatToConfigs.begin(); it != formatToConfigs.end();) {
> > +                     auto& streamConfigs = it->second;
> > +                     if (streamConfigs.empty()) {
> > +                             it = formatToConfigs.erase(it);
> > +                             continue;
> > +                     }
> > +                     sortedStreamConfigs.push_back(streamConfigs.back());
> > +                     streamConfigs.pop_back();
> > +                     it++;
>
> AH! this it++ was not here in the previous version :)
> Now I see what you wanted.
>
> This makes me think that a LOG(Debug) printout of the the sorted
> stream configs would help making sure we obtain what we intend.
>

I somehow missed it++ in the previous patch. Sorry!

Regards,
-Hiro
>
> > +             }
> > +     }
> > +     assert(sortedStreamConfigs.size() == unsortedStreamConfigsSize);
> > +
> > +     return sortedStreamConfigs;
> > +}
> > +} /* namespace */
> >
> >  MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer,
> >                                        int flags)
> > @@ -1333,6 +1431,8 @@ int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)
> >               streamConfigs[index].types.push_back(type);
> >       }
> >
> > +     streamConfigs = sortCamera3StreamConfigs(std::move(streamConfigs),
> > +                                               jpegStream);
> >       for (const auto &streamConfig : streamConfigs) {
> >               config_->addConfiguration(streamConfig.config);
> >               for (size_t i = 0; i < streamConfig.streams.size(); ++i) {
> > --
> > 2.29.2.576.ga3fc446d84-goog

Patch
diff mbox series 

diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index b7bf3d88..dcba4e7d 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -9,6 +9,7 @@ 
 #include "camera_ops.h"
 #include "post_processor.h"

+#include <optional>
 #include <sys/mman.h>
 #include <tuple>
 #include <vector>
@@ -27,6 +28,8 @@ 

 using namespace libcamera;

+LOG_DECLARE_CATEGORY(HAL)
+
 namespace {

 /*
@@ -140,9 +143,104 @@  struct Camera3StreamConfig {
 	std::vector<CameraStream::Type> types;
 	StreamConfiguration config;
 };
-} /* namespace */

-LOG_DECLARE_CATEGORY(HAL)
+/*
+ * Reorder the configurations so that CameraDevice can accept them as much as
+ * possible. The sort rule is as follows.
+ * 1.) The configuration for NV12 request whose resolution is the largest.
+ * 2.) The configuration for JPEG request.
+ * 3.) Others. Larger resolutions and different formats are put earlier.
+ */
+std::vector<Camera3StreamConfig> sortCamera3StreamConfigs(
+	std::vector<Camera3StreamConfig> unsortedStreamConfigs,
+	const camera3_stream_t *jpegStream) {
+	const size_t unsortedStreamConfigsSize = unsortedStreamConfigs.size();
+	std::optional<Camera3StreamConfig> streamConfigForJpeg = std::nullopt;
+	if (jpegStream) {
+		for (auto it = unsortedStreamConfigs.begin();
+		     it != unsortedStreamConfigs.end(); it++) {
+			const auto &streams = it->streams;
+			if (std::find(streams.begin(), streams.end(),
+				      jpegStream) != streams.end()) {
+				streamConfigForJpeg = *it;
+				unsortedStreamConfigs.erase(it);
+				break;
+			}
+		}
+                if (!streamConfigForJpeg)
+                  LOG(HAL, Fatal) << "No Camera3StreamConfig is found for Jpeg";
+	}
+
+	std::map<uint32_t, std::vector<Camera3StreamConfig>> formatToConfigs;
+	for (const auto &streamConfig : unsortedStreamConfigs) {
+		const StreamConfiguration &config = streamConfig.config;
+		formatToConfigs[config.pixelFormat].push_back(streamConfig);
+
+	}
+	for (auto& [format, streamConfigs] : formatToConfigs) {
+		/* Sorted by resolution. Smaller is put first. */
+		std::sort(streamConfigs.begin(), streamConfigs.end(),
+			  [](const auto &streamConfigA, const auto &streamConfigB) {
+				  const Size &sizeA = streamConfigA.config.size;
+				  const Size &sizeB = streamConfigB.config.size;
+                                  return sizeA < sizeB;
+			  });
+	}
+
+	std::vector<Camera3StreamConfig> sortedStreamConfigs;
+        sortedStreamConfigs.reserve(unsortedStreamConfigsSize);
+	/*
+	 * NV12 is the most prioritized format. Put the configuration with NV12
+	 * and the largest resolution first.
+	 */
+	const auto nv12Stream = formatToConfigs.find(formats::NV12);
+	if (nv12Stream != formatToConfigs.end()) {
+		auto &nv12StreamConfigs = nv12Stream->second;
+		const Size &nv12LargestSize = nv12StreamConfigs.back().config.size;
+		/*
+		 * If JPEG will be created from NV12 and the size is larger than
+		 * the largest NV12 configurations, then put the NV12
+		 * configuration for JPEG first.
+		 */
+		if (streamConfigForJpeg &&
+		    streamConfigForJpeg->config.pixelFormat == formats::NV12) {
+			const Size &nv12SizeForJpeg = streamConfigForJpeg->config.size;
+			if (nv12LargestSize < nv12SizeForJpeg) {
+				sortedStreamConfigs.push_back(*streamConfigForJpeg);
+				streamConfigForJpeg = std::nullopt;
+			}
+		}
+		sortedStreamConfigs.push_back(nv12StreamConfigs.back());
+		nv12StreamConfigs.pop_back();
+	}
+
+	/* If the configuration for JPEG is there, then put it. */
+	if (streamConfigForJpeg) {
+		sortedStreamConfigs.push_back(*streamConfigForJpeg);
+		streamConfigForJpeg = std::nullopt;
+	}
+
+	/*
+	 * Put configurations with different formats and larger resolutions
+	 * earlier.
+	 */
+	while (!formatToConfigs.empty()) {
+		for (auto it = formatToConfigs.begin(); it != formatToConfigs.end();) {
+			auto& streamConfigs = it->second;
+			if (streamConfigs.empty()) {
+				it = formatToConfigs.erase(it);
+				continue;
+			}
+			sortedStreamConfigs.push_back(streamConfigs.back());
+			streamConfigs.pop_back();
+			it++;
+		}
+	}
+	assert(sortedStreamConfigs.size() == unsortedStreamConfigsSize);
+
+	return sortedStreamConfigs;
+}
+} /* namespace */

 MappedCamera3Buffer::MappedCamera3Buffer(const buffer_handle_t camera3buffer,
 					 int flags)
@@ -1333,6 +1431,8 @@  int CameraDevice::configureStreams(camera3_stream_configuration_t *stream_list)
 		streamConfigs[index].types.push_back(type);
 	}

+	streamConfigs = sortCamera3StreamConfigs(std::move(streamConfigs),
+						  jpegStream);
 	for (const auto &streamConfig : streamConfigs) {
 		config_->addConfiguration(streamConfig.config);
 		for (size_t i = 0; i < streamConfig.streams.size(); ++i) {