[v2,1/9] ipa: libipa: Add generic Interpolator class

21250 diff mbox series

The MatrixInterpolator is great for interpolation of matrices for
different color temperatures. It has however one limitation - it can
only handle matrices. For LSC it would be great to interpolate the LSC
tables (or even polynomials) using the same approach. Add a generic
Interpolator class based on the existing MatrixInterpolator. This calss
can be adapted to any other type using partial template specialization.

Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
---
 src/ipa/libipa/interpolator.cpp | 156 ++++++++++++++++++++++++++++++++
 src/ipa/libipa/interpolator.h   | 131 +++++++++++++++++++++++++++
 src/ipa/libipa/meson.build      |   2 +
 3 files changed, 289 insertions(+)
 create mode 100644 src/ipa/libipa/interpolator.cpp
 create mode 100644 src/ipa/libipa/interpolator.h

Quoting Stefan Klug (2024-09-13 08:57:19)
> The MatrixInterpolator is great for interpolation of matrices for
> different color temperatures. It has however one limitation - it can
> only handle matrices. For LSC it would be great to interpolate the LSC
> tables (or even polynomials) using the same approach. Add a generic
> Interpolator class based on the existing MatrixInterpolator. This calss

s/calss/class/

> can be adapted to any other type using partial template specialization.
> 
> Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
> ---
>  src/ipa/libipa/interpolator.cpp | 156 ++++++++++++++++++++++++++++++++
>  src/ipa/libipa/interpolator.h   | 131 +++++++++++++++++++++++++++
>  src/ipa/libipa/meson.build      |   2 +
>  3 files changed, 289 insertions(+)
>  create mode 100644 src/ipa/libipa/interpolator.cpp
>  create mode 100644 src/ipa/libipa/interpolator.h
> 
> diff --git a/src/ipa/libipa/interpolator.cpp b/src/ipa/libipa/interpolator.cpp
> new file mode 100644
> index 000000000000..05a4af984f9a
> --- /dev/null
> +++ b/src/ipa/libipa/interpolator.cpp
> @@ -0,0 +1,156 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
> + *
> + * Helper class for interpolating objects
> + */
> +#include "interpolator.h"
> +
> +#include <algorithm>
> +#include <string>
> +
> +#include <libcamera/base/log.h>
> +
> +#include "libcamera/internal/yaml_parser.h"
> +
> +#include "interpolator.h"
> +
> +/**
> + * \file interpolator.h
> + * \brief Helper class for linear interpolating a set of objects
> + */
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(Interpolator)
> +
> +namespace ipa {
> +
> +/**
> + * \class Interpolator
> + * \brief Class for storing, retrieving, and interpolating objects
> + * \tparam T Type of objects stored in the interpolator
> + *
> + * The main use case is to pass a map from color temperatures to corresponding
> + * objects (eg. matrices for color correction), and then requesting a
> + * interpolated object for a specific color temperature. This class will
> + * abstract away the interpolation portion.
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator()
> + * \brief Construct a empty interpolator
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator(const std::map<unsigned int, T> &data)
> + * \brief Construct a interpolator from a map of objects
> + * \param data Map from which to construct the interpolator
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator(std::map<unsigned int, T> &&data)
> + * \brief Construct a interpolator from a map of objects

s/a interpolator/an interpolator/

> + * \param data Map from which to construct the interpolator
> + */
> +
> +/**
> + * \fn int Interpolator<T>::readYaml(const libcamera::YamlObject &yaml,
> +                                    const std::string &key_name,
> +                                    const std::string &value_name)
> + * \brief Initialize an Interpolator instance from yaml
> + * \tparam T Type of data stored in the interpolator
> + * \param[in] yaml The yaml object that contains the map of unsigned integers to objects
> + * \param[in] key_name The name of the key in the yaml object
> + * \param[in] value_name The name of the value in the yaml object
> + *
> + * The yaml object is expected to be a list of maps. Each map has two or more
> + * pairs: one of \a key_name to the key value (usually color temperature), and
> + * one or more of \a value_name to the object. This is a bit difficult to
> + * explain, so here is an example (in python, as it is easier to parse than
> + * yaml):
> + *       [
> + *               {
> + *                   'ct': 2860,
> + *                   'ccm': [ 2.12089, -0.52461, -0.59629,
> + *                           -0.85342,  2.80445, -0.95103,
> + *                           -0.26897, -1.14788,  2.41685 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *
> + *               {
> + *                   'ct': 2960,
> + *                   'ccm': [ 2.26962, -0.54174, -0.72789,
> + *                           -0.77008,  2.60271, -0.83262,
> + *                           -0.26036, -1.51254,  2.77289 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *
> + *               {
> + *                   'ct': 3603,
> + *                   'ccm': [ 2.18644, -0.66148, -0.52496,
> + *                           -0.77828,  2.69474, -0.91645,
> + *                           -0.25239, -0.83059,  2.08298 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *       ]
> + *
> + * In this case, \a key_name would be 'ct', and \a value_name can be either
> + * 'ccm' or 'offsets'. This way multiple interpolators can be defined in
> + * one set of color temperature ranges in the tuning file, and they can be
> + * retrieved separately with the \a value_name parameter.
> + *
> + * \return Zero on success, negative error code otherwise
> + */
> +
> +/**
> + * \fn void Interpolator<T>::setQuantization(const unsigned int q)
> + * \brief Set the quantization value
> + * \param[in] q The quantization value
> + *
> + * Sets the quantization value. When this is set, 'key' gets quantized to this
> + * size, before doing the interpolation. This can help in reducing the number of
> + * updated pushed to the hardware.

s/updated/updates/


And that's all I can find so ...


Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>

> + *
> + * Note that normally a threshold needs to be combined with quantization.
> + * Otherwise a value that swings around the edge of the quantization step will
> + * lead to constant updates.
> + */
> +
> +/**
> + * \fn void Interpolator<T>::setData(std::map<unsigned int, T> &&data)
> + * \brief Set the internal map
> + *
> + * Overwrites the internal map using move semantics.
> + */
> +
> +/**
> + * \fn const T& Interpolator<T>::getInterpolated(unsigned int key, unsigned int *quantizedKey = nullptr)
> + * \brief Retrieve a a interpolated value for the given key
> + * \param[in] key The unsigned integer key of the object to retrieve
> + * \param[out] quantizedKey If provided, the key value after quantization
> + * \return The object corresponding to the key. The object is cached internally,
> + * so on successive calls with the same key (after quantization) interpolation
> + * is not recalculated.
> + */
> +
> +/**
> + * \fn void Interpolator<T>::interpolate(const T &a, const T &b, T &dest, double
> + * lambda)
> + * \brief Interpolate between two instances of T
> + * \param a The first value to interpolate
> + * \param b The second value to interpolate
> + * \param dest The destination for the interpolated value
> + * \param lambda The interpolation factor (0..1)
> + *
> + * Interpolates between a and b according to lambda. It calculates dest = a *
> + * (1.0 - lambda) + b * lambda;
> + *
> + * If T supports multiplication with double and addition, this function can be
> + * used as is. For other types this function can be overwritten using partial
> + * template specialization.
> + */
> +
> +} /* namespace ipa */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/libipa/interpolator.h b/src/ipa/libipa/interpolator.h
> new file mode 100644
> index 000000000000..fffce21465fe
> --- /dev/null
> +++ b/src/ipa/libipa/interpolator.h
> @@ -0,0 +1,131 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
> + *
> + * Helper class for interpolating maps of objects
> + */
> +
> +#pragma once
> +
> +#include <algorithm>
> +#include <cmath>
> +#include <map>
> +#include <string>
> +#include <tuple>
> +
> +#include <libcamera/base/log.h>
> +
> +#include "libcamera/internal/yaml_parser.h"
> +
> +namespace libcamera {
> +
> +LOG_DECLARE_CATEGORY(Interpolator)
> +
> +namespace ipa {
> +
> +template<typename T>
> +class Interpolator
> +{
> +public:
> +       Interpolator() = default;
> +       Interpolator(const std::map<unsigned int, T> &data)
> +               : data_(data)
> +       {
> +       }
> +       Interpolator(std::map<unsigned int, T> &&data)
> +               : data_(std::move(data))
> +       {
> +       }
> +
> +       ~Interpolator() = default;
> +
> +       int readYaml(const libcamera::YamlObject &yaml,
> +                    const std::string &key_name,
> +                    const std::string &value_name)
> +       {
> +               data_.clear();
> +               lastInterpolatedKey_.reset();
> +
> +               if (!yaml.isList()) {
> +                       LOG(Interpolator, Error) << "yaml object must be a list";
> +                       return -EINVAL;
> +               }
> +
> +               for (const auto &value : yaml.asList()) {
> +                       unsigned int ct = std::stoul(value[key_name].get<std::string>(""));
> +                       std::optional<T> data =
> +                               value[value_name].get<T>();
> +                       if (!data) {
> +                               return -EINVAL;
> +                       }
> +
> +                       data_[ct] = *data;
> +               }
> +
> +               if (data_.size() < 1) {
> +                       LOG(Interpolator, Error) << "Need at least one element";
> +                       return -EINVAL;
> +               }
> +
> +               return 0;
> +       }
> +
> +       void setQuantization(const unsigned int q)
> +       {
> +               quantization_ = q;
> +       }
> +
> +       void setData(std::map<unsigned int, T> &&data)
> +       {
> +               data_ = std::move(data);
> +               lastInterpolatedKey_.reset();
> +       }
> +
> +       const T &getInterpolated(unsigned int key, unsigned int *quantizedKey = nullptr)
> +       {
> +               ASSERT(data_.size() > 0);
> +
> +               if (quantization_ > 0)
> +                       key = std::lround(key / static_cast<double>(quantization_)) * quantization_;
> +
> +               if (quantizedKey)
> +                       *quantizedKey = key;
> +
> +               if (lastInterpolatedKey_.has_value() &&
> +                   *lastInterpolatedKey_ == key)
> +                       return lastInterpolatedValue_;
> +
> +               auto it = data_.lower_bound(key);
> +
> +               if (it == data_.begin())
> +                       return it->second;
> +
> +               if (it == data_.end())
> +                       return std::prev(it)->second;
> +
> +               if (it->first == key)
> +                       return it->second;
> +
> +               auto it2 = std::prev(it);
> +               double lambda = (key - it2->first) / static_cast<double>(it->first - it2->first);
> +               interpolate(it2->second, it->second, lastInterpolatedValue_, lambda);
> +               lastInterpolatedKey_ = key;
> +
> +               return lastInterpolatedValue_;
> +       }
> +
> +       void interpolate(const T &a, const T &b, T &dest, double lambda)
> +       {
> +               dest = a * (1.0 - lambda) + b * lambda;
> +       }
> +
> +private:
> +       std::map<unsigned int, T> data_;
> +       T lastInterpolatedValue_;
> +       std::optional<unsigned int> lastInterpolatedKey_;
> +       unsigned int quantization_ = 0;
> +};
> +
> +} /* namespace ipa */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build
> index eff8ce2660f1..2c2712a7d252 100644
> --- a/src/ipa/libipa/meson.build
> +++ b/src/ipa/libipa/meson.build
> @@ -7,6 +7,7 @@ libipa_headers = files([
>      'exposure_mode_helper.h',
>      'fc_queue.h',
>      'histogram.h',
> +    'interpolator.h',
>      'matrix.h',
>      'matrix_interpolator.h',
>      'module.h',
> @@ -21,6 +22,7 @@ libipa_sources = files([
>      'exposure_mode_helper.cpp',
>      'fc_queue.cpp',
>      'histogram.cpp',
> +    'interpolator.cpp',
>      'matrix.cpp',
>      'matrix_interpolator.cpp',
>      'module.cpp',
> -- 
> 2.43.0
>

On Fri, Sep 13, 2024 at 09:57:19AM +0200, Stefan Klug wrote:
> The MatrixInterpolator is great for interpolation of matrices for
> different color temperatures. It has however one limitation - it can
> only handle matrices. For LSC it would be great to interpolate the LSC
> tables (or even polynomials) using the same approach. Add a generic
> Interpolator class based on the existing MatrixInterpolator. This calss
> can be adapted to any other type using partial template specialization.
> 
> Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>

Kieran beat me to it but "an interpolator".

Other than that,

Reviewed-by: Paul Elder <paul.elder@ideasonboard.com>

> ---
>  src/ipa/libipa/interpolator.cpp | 156 ++++++++++++++++++++++++++++++++
>  src/ipa/libipa/interpolator.h   | 131 +++++++++++++++++++++++++++
>  src/ipa/libipa/meson.build      |   2 +
>  3 files changed, 289 insertions(+)
>  create mode 100644 src/ipa/libipa/interpolator.cpp
>  create mode 100644 src/ipa/libipa/interpolator.h
> 
> diff --git a/src/ipa/libipa/interpolator.cpp b/src/ipa/libipa/interpolator.cpp
> new file mode 100644
> index 000000000000..05a4af984f9a
> --- /dev/null
> +++ b/src/ipa/libipa/interpolator.cpp
> @@ -0,0 +1,156 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
> + *
> + * Helper class for interpolating objects
> + */
> +#include "interpolator.h"
> +
> +#include <algorithm>
> +#include <string>
> +
> +#include <libcamera/base/log.h>
> +
> +#include "libcamera/internal/yaml_parser.h"
> +
> +#include "interpolator.h"
> +
> +/**
> + * \file interpolator.h
> + * \brief Helper class for linear interpolating a set of objects
> + */
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(Interpolator)
> +
> +namespace ipa {
> +
> +/**
> + * \class Interpolator
> + * \brief Class for storing, retrieving, and interpolating objects
> + * \tparam T Type of objects stored in the interpolator
> + *
> + * The main use case is to pass a map from color temperatures to corresponding
> + * objects (eg. matrices for color correction), and then requesting a
> + * interpolated object for a specific color temperature. This class will
> + * abstract away the interpolation portion.
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator()
> + * \brief Construct a empty interpolator
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator(const std::map<unsigned int, T> &data)
> + * \brief Construct a interpolator from a map of objects
> + * \param data Map from which to construct the interpolator
> + */
> +
> +/**
> + * \fn Interpolator::Interpolator(std::map<unsigned int, T> &&data)
> + * \brief Construct a interpolator from a map of objects
> + * \param data Map from which to construct the interpolator
> + */
> +
> +/**
> + * \fn int Interpolator<T>::readYaml(const libcamera::YamlObject &yaml,
> +		                     const std::string &key_name,
> +		                     const std::string &value_name)
> + * \brief Initialize an Interpolator instance from yaml
> + * \tparam T Type of data stored in the interpolator
> + * \param[in] yaml The yaml object that contains the map of unsigned integers to objects
> + * \param[in] key_name The name of the key in the yaml object
> + * \param[in] value_name The name of the value in the yaml object
> + *
> + * The yaml object is expected to be a list of maps. Each map has two or more
> + * pairs: one of \a key_name to the key value (usually color temperature), and
> + * one or more of \a value_name to the object. This is a bit difficult to
> + * explain, so here is an example (in python, as it is easier to parse than
> + * yaml):
> + *       [
> + *               {
> + *                   'ct': 2860,
> + *                   'ccm': [ 2.12089, -0.52461, -0.59629,
> + *                           -0.85342,  2.80445, -0.95103,
> + *                           -0.26897, -1.14788,  2.41685 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *
> + *               {
> + *                   'ct': 2960,
> + *                   'ccm': [ 2.26962, -0.54174, -0.72789,
> + *                           -0.77008,  2.60271, -0.83262,
> + *                           -0.26036, -1.51254,  2.77289 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *
> + *               {
> + *                   'ct': 3603,
> + *                   'ccm': [ 2.18644, -0.66148, -0.52496,
> + *                           -0.77828,  2.69474, -0.91645,
> + *                           -0.25239, -0.83059,  2.08298 ],
> + *                   'offsets': [ 0, 0, 0 ]
> + *               },
> + *       ]
> + *
> + * In this case, \a key_name would be 'ct', and \a value_name can be either
> + * 'ccm' or 'offsets'. This way multiple interpolators can be defined in
> + * one set of color temperature ranges in the tuning file, and they can be
> + * retrieved separately with the \a value_name parameter.
> + *
> + * \return Zero on success, negative error code otherwise
> + */
> +
> +/**
> + * \fn void Interpolator<T>::setQuantization(const unsigned int q)
> + * \brief Set the quantization value
> + * \param[in] q The quantization value
> + *
> + * Sets the quantization value. When this is set, 'key' gets quantized to this
> + * size, before doing the interpolation. This can help in reducing the number of
> + * updated pushed to the hardware.
> + *
> + * Note that normally a threshold needs to be combined with quantization.
> + * Otherwise a value that swings around the edge of the quantization step will
> + * lead to constant updates.
> + */
> +
> +/**
> + * \fn void Interpolator<T>::setData(std::map<unsigned int, T> &&data)
> + * \brief Set the internal map
> + *
> + * Overwrites the internal map using move semantics.
> + */
> +
> +/**
> + * \fn const T& Interpolator<T>::getInterpolated(unsigned int key, unsigned int *quantizedKey = nullptr)
> + * \brief Retrieve a a interpolated value for the given key
> + * \param[in] key The unsigned integer key of the object to retrieve
> + * \param[out] quantizedKey If provided, the key value after quantization
> + * \return The object corresponding to the key. The object is cached internally,
> + * so on successive calls with the same key (after quantization) interpolation
> + * is not recalculated.
> + */
> +
> +/**
> + * \fn void Interpolator<T>::interpolate(const T &a, const T &b, T &dest, double
> + * lambda)
> + * \brief Interpolate between two instances of T
> + * \param a The first value to interpolate
> + * \param b The second value to interpolate
> + * \param dest The destination for the interpolated value
> + * \param lambda The interpolation factor (0..1)
> + *
> + * Interpolates between a and b according to lambda. It calculates dest = a *
> + * (1.0 - lambda) + b * lambda;
> + *
> + * If T supports multiplication with double and addition, this function can be
> + * used as is. For other types this function can be overwritten using partial
> + * template specialization.
> + */
> +
> +} /* namespace ipa */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/libipa/interpolator.h b/src/ipa/libipa/interpolator.h
> new file mode 100644
> index 000000000000..fffce21465fe
> --- /dev/null
> +++ b/src/ipa/libipa/interpolator.h
> @@ -0,0 +1,131 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
> + *
> + * Helper class for interpolating maps of objects
> + */
> +
> +#pragma once
> +
> +#include <algorithm>
> +#include <cmath>
> +#include <map>
> +#include <string>
> +#include <tuple>
> +
> +#include <libcamera/base/log.h>
> +
> +#include "libcamera/internal/yaml_parser.h"
> +
> +namespace libcamera {
> +
> +LOG_DECLARE_CATEGORY(Interpolator)
> +
> +namespace ipa {
> +
> +template<typename T>
> +class Interpolator
> +{
> +public:
> +	Interpolator() = default;
> +	Interpolator(const std::map<unsigned int, T> &data)
> +		: data_(data)
> +	{
> +	}
> +	Interpolator(std::map<unsigned int, T> &&data)
> +		: data_(std::move(data))
> +	{
> +	}
> +
> +	~Interpolator() = default;
> +
> +	int readYaml(const libcamera::YamlObject &yaml,
> +		     const std::string &key_name,
> +		     const std::string &value_name)
> +	{
> +		data_.clear();
> +		lastInterpolatedKey_.reset();
> +
> +		if (!yaml.isList()) {
> +			LOG(Interpolator, Error) << "yaml object must be a list";
> +			return -EINVAL;
> +		}
> +
> +		for (const auto &value : yaml.asList()) {
> +			unsigned int ct = std::stoul(value[key_name].get<std::string>(""));
> +			std::optional<T> data =
> +				value[value_name].get<T>();
> +			if (!data) {
> +				return -EINVAL;
> +			}
> +
> +			data_[ct] = *data;
> +		}
> +
> +		if (data_.size() < 1) {
> +			LOG(Interpolator, Error) << "Need at least one element";
> +			return -EINVAL;
> +		}
> +
> +		return 0;
> +	}
> +
> +	void setQuantization(const unsigned int q)
> +	{
> +		quantization_ = q;
> +	}
> +
> +	void setData(std::map<unsigned int, T> &&data)
> +	{
> +		data_ = std::move(data);
> +		lastInterpolatedKey_.reset();
> +	}
> +
> +	const T &getInterpolated(unsigned int key, unsigned int *quantizedKey = nullptr)
> +	{
> +		ASSERT(data_.size() > 0);
> +
> +		if (quantization_ > 0)
> +			key = std::lround(key / static_cast<double>(quantization_)) * quantization_;
> +
> +		if (quantizedKey)
> +			*quantizedKey = key;
> +
> +		if (lastInterpolatedKey_.has_value() &&
> +		    *lastInterpolatedKey_ == key)
> +			return lastInterpolatedValue_;
> +
> +		auto it = data_.lower_bound(key);
> +
> +		if (it == data_.begin())
> +			return it->second;
> +
> +		if (it == data_.end())
> +			return std::prev(it)->second;
> +
> +		if (it->first == key)
> +			return it->second;
> +
> +		auto it2 = std::prev(it);
> +		double lambda = (key - it2->first) / static_cast<double>(it->first - it2->first);
> +		interpolate(it2->second, it->second, lastInterpolatedValue_, lambda);
> +		lastInterpolatedKey_ = key;
> +
> +		return lastInterpolatedValue_;
> +	}
> +
> +	void interpolate(const T &a, const T &b, T &dest, double lambda)
> +	{
> +		dest = a * (1.0 - lambda) + b * lambda;
> +	}
> +
> +private:
> +	std::map<unsigned int, T> data_;
> +	T lastInterpolatedValue_;
> +	std::optional<unsigned int> lastInterpolatedKey_;
> +	unsigned int quantization_ = 0;
> +};
> +
> +} /* namespace ipa */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build
> index eff8ce2660f1..2c2712a7d252 100644
> --- a/src/ipa/libipa/meson.build
> +++ b/src/ipa/libipa/meson.build
> @@ -7,6 +7,7 @@ libipa_headers = files([
>      'exposure_mode_helper.h',
>      'fc_queue.h',
>      'histogram.h',
> +    'interpolator.h',
>      'matrix.h',
>      'matrix_interpolator.h',
>      'module.h',
> @@ -21,6 +22,7 @@ libipa_sources = files([
>      'exposure_mode_helper.cpp',
>      'fc_queue.cpp',
>      'histogram.cpp',
> +    'interpolator.cpp',
>      'matrix.cpp',
>      'matrix_interpolator.cpp',
>      'module.cpp',
> -- 
> 2.43.0
>