[libcamera-devel,v5,06/10] ipa: rkisp1: Add AF algorithm based on AfHillClimbing
diff mbox series

Message ID 20230324142908.64224-7-dse@thaumatec.com
State Superseded
Headers show
Series
  • ipa: rkisp1: Add autofocus algorithm
Related show

Commit Message

Daniel Semkowicz March 24, 2023, 2:29 p.m. UTC 
Add contrast based auto focus implementation for rkisp1 platform using
the common AfHillClimbing algorithm.

Rockchip ISP AF block allows calculation of sharpness and luminance
in up to three user defined windows. If no windows are set, there are
some default settings applied for the first window and exposed through
the driver. For each frame, use the sharpness value calculated for this
default window and feed the hill climbing algorithm with them. Then set
the lens position to value calculated by the algorithm.

Signed-off-by: Daniel Semkowicz <dse@thaumatec.com>
---
 src/ipa/rkisp1/algorithms/af.cpp      | 117 ++++++++++++++++++++++++++
 src/ipa/rkisp1/algorithms/af.h        |  43 ++++++++++
 src/ipa/rkisp1/algorithms/meson.build |   1 +
 src/ipa/rkisp1/ipa_context.cpp        |  13 +++
 src/ipa/rkisp1/ipa_context.h          |   5 ++
 5 files changed, 179 insertions(+)
 create mode 100644 src/ipa/rkisp1/algorithms/af.cpp
 create mode 100644 src/ipa/rkisp1/algorithms/af.h

Comments

Jacopo Mondi March 24, 2023, 3:39 p.m. UTC | #1 
Hi Daniel

On Fri, Mar 24, 2023 at 03:29:04PM +0100, Daniel Semkowicz via libcamera-devel wrote:
> Add contrast based auto focus implementation for rkisp1 platform using
> the common AfHillClimbing algorithm.
>
> Rockchip ISP AF block allows calculation of sharpness and luminance
> in up to three user defined windows. If no windows are set, there are
> some default settings applied for the first window and exposed through
> the driver. For each frame, use the sharpness value calculated for this
> default window and feed the hill climbing algorithm with them. Then set
> the lens position to value calculated by the algorithm.
>
> Signed-off-by: Daniel Semkowicz <dse@thaumatec.com>
> ---
>  src/ipa/rkisp1/algorithms/af.cpp      | 117 ++++++++++++++++++++++++++
>  src/ipa/rkisp1/algorithms/af.h        |  43 ++++++++++
>  src/ipa/rkisp1/algorithms/meson.build |   1 +
>  src/ipa/rkisp1/ipa_context.cpp        |  13 +++
>  src/ipa/rkisp1/ipa_context.h          |   5 ++
>  5 files changed, 179 insertions(+)
>  create mode 100644 src/ipa/rkisp1/algorithms/af.cpp
>  create mode 100644 src/ipa/rkisp1/algorithms/af.h
>
> diff --git a/src/ipa/rkisp1/algorithms/af.cpp b/src/ipa/rkisp1/algorithms/af.cpp
> new file mode 100644
> index 00000000..aca127d2
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/af.cpp
> @@ -0,0 +1,117 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2023, Theobroma Systems
> + *
> + * af.cpp - RkISP1 AF hill climbing based control algorithm
> + */
> +
> +#include "af.h"
> +
> +/**
> + * \file af.h
> + */
> +
> +namespace libcamera {
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +/**
> + * \class Af
> + * \brief AF contrast based hill climbing control algorithm for RkISP platforms
> + *
> + * Auto focus algorithm for RkISP platforms, based on the common hill climbing
> + * auto focus implementation
> + * (libcamera::ipa::algorithms::AfHillClimbing).

fits on the previous line

> + *
> + * This is the platform layer of the algorithm.
> + *
> + * Tuning file parameters:
> + * - **wait-frames-lens:** Number of frames that should be skipped when lens
> + *   position is changed. Lens movement takes some time and statistics measured
> + *   during the lens movement are unstable. Currently there is no way to know
> + *   when lens movement finished and this is a workaround for this. Wait a fixed
> + *   amount of time on each movement. This parameter should be set according
> + *   to the worst case  - the number of frames it takes to move lens between
> + *   limit positions.
> + * .
> + * \sa libcamera::ipa::algorithms::AfHillClimbing for additional tuning
> + * parameters.
> + *
> + * \todo Model the lens delay as number of frames required for the lens position
> + *   to stabilize in the CameraLens class.
      ^ rogue space

Thanks :)

> + */
> +
> +LOG_DEFINE_CATEGORY(RkISP1Af)
> +
> +/**
> + * \copydoc libcamera::ipa::Algorithm::init
> + */
> +int Af::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData)
> +{
> +	waitFramesLens_ = tuningData["wait-frames-lens"].get<uint32_t>(1);
> +
> +	LOG(RkISP1Af, Debug) << "waitFramesLens_: " << waitFramesLens_;
> +
> +	return af.init(tuningData);
> +}
> +
> +/**
> + * \copydoc libcamera::ipa::Algorithm::configure
> + */
> +int Af::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
> +{
> +	const auto &lensConfig = context.configuration.lens;
> +
> +	af.configure(lensConfig.minFocusPosition, lensConfig.maxFocusPosition,
> +		     configInfo.outputSize);
> +
> +	/*
> +	 * Lens position is unknown at the startup, so initialize
> +	 * the current position to something out of range.
> +	 */
> +	context.activeState.af.lensPosition = lensConfig.maxFocusPosition + 1;
> +
> +	return 0;
> +}
> +
> +/**
> + * \copydoc libcamera::ipa::Algorithm::queueRequest
> + */
> +void Af::queueRequest([[maybe_unused]] IPAContext &context,
> +		      [[maybe_unused]] const uint32_t frame,
> +		      [[maybe_unused]] IPAFrameContext &frameContext,
> +		      const ControlList &controls)
> +{
> +	af.queueRequest(controls);
> +}
> +
> +/**
> + * \copydoc libcamera::ipa::Algorithm::process
> + */
> +void Af::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
> +		 [[maybe_unused]] IPAFrameContext &frameContext,
> +		 const rkisp1_stat_buffer *stats,
> +		 [[maybe_unused]] ControlList &metadata)
> +{
> +	uint32_t sharpness = stats->params.af.window[0].sum;
> +	uint32_t luminance = stats->params.af.window[0].lum;
> +
> +	LOG(RkISP1Af, Debug)
> +		<< "lensPosition: " << context.activeState.af.lensPosition
> +		<< ", Sharpness: " << sharpness
> +		<< ", Luminance: " << luminance;
> +
> +	int32_t newLensPosition = af.process(sharpness);
> +
> +	if (newLensPosition != context.activeState.af.lensPosition) {
> +		context.activeState.af.lensPosition = newLensPosition;
> +		context.activeState.af.applyLensCtrls = true;
> +		af.setFramesToSkip(waitFramesLens_);

I wonder if this wouldn't be better names simply "skipFrames()"

> +	}
> +}
> +
> +REGISTER_IPA_ALGORITHM(Af, "Af")
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/rkisp1/algorithms/af.h b/src/ipa/rkisp1/algorithms/af.h
> new file mode 100644
> index 00000000..3ba66d38
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/af.h
> @@ -0,0 +1,43 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2023, Theobroma Systems
> + *
> + * af.h - RkISP1 AF hill climbing based control algorithm
> + */
> +
> +#pragma once
> +
> +#include <linux/rkisp1-config.h>
> +
> +#include "libipa/algorithms/af_hill_climbing.h"
> +
> +#include "algorithm.h"
> +
> +namespace libcamera {
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +class Af : public Algorithm
> +{
> +public:
> +	int init(IPAContext &context, const YamlObject &tuningData) override;
> +	int configure(IPAContext &context,
> +		      const IPACameraSensorInfo &configInfo) override;
> +	void queueRequest(IPAContext &context, uint32_t frame,
> +			  IPAFrameContext &frameContext,
> +			  const ControlList &controls) override;
> +	void process(IPAContext &context, uint32_t frame,
> +		     IPAFrameContext &frameContext,
> +		     const rkisp1_stat_buffer *stats,
> +		     ControlList &metadata) override;
> +
> +private:
> +	ipa::algorithms::AfHillClimbing af;

I envision one day where this can be made
        ipa::algorithms::Af *af;

and the actual algorithm instantiated a run-time based on a
configuration option. This is a good start though

> +
> +	/* Wait number of frames after changing lens position */
> +	uint32_t waitFramesLens_ = 0;
> +};
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build
> index 93a48329..ab7e44f3 100644
> --- a/src/ipa/rkisp1/algorithms/meson.build
> +++ b/src/ipa/rkisp1/algorithms/meson.build
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: CC0-1.0
>
>  rkisp1_ipa_algorithms = files([
> +    'af.cpp',
>      'agc.cpp',
>      'awb.cpp',
>      'blc.cpp',
> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
> index 401c098f..5c5f80c7 100644
> --- a/src/ipa/rkisp1/ipa_context.cpp
> +++ b/src/ipa/rkisp1/ipa_context.cpp
> @@ -134,6 +134,19 @@ namespace libcamera::ipa::rkisp1 {
>   * member may be read by any algorithm, but shall only be written by its owner.
>   */
>
> +/**
> + * \var IPAActiveState::af
> + * \brief State for the Automatic Focus Control algorithm
> + *
> + * \var IPAActiveState::af.lensPosition
> + * \brief Lens position calculated by the AF algorithm
> + *
> + * \var IPAActiveState::af.applyLensCtrls
> + * \brief Whether the lens position should be applied
> + *
> + * If true, IPA should send new controls to the PH to set new lens position.
> + */
> +
>  /**
>   * \var IPAActiveState::agc
>   * \brief State for the Automatic Gain Control algorithm
> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
> index bfb6e1b7..2c2eec3b 100644
> --- a/src/ipa/rkisp1/ipa_context.h
> +++ b/src/ipa/rkisp1/ipa_context.h
> @@ -58,6 +58,11 @@ struct IPASessionConfiguration {
>  };
>
>  struct IPAActiveState {
> +	struct {
> +		int32_t lensPosition;
> +		bool applyLensCtrls;
> +	} af;
> +

Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>

Thanks
  j

>  	struct {
>  		struct {
>  			uint32_t exposure;
> --
> 2.39.2
>

Patch
diff mbox series 

diff --git a/src/ipa/rkisp1/algorithms/af.cpp b/src/ipa/rkisp1/algorithms/af.cpp
new file mode 100644
index 00000000..aca127d2
--- /dev/null
+++ b/src/ipa/rkisp1/algorithms/af.cpp
@@ -0,0 +1,117 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2023, Theobroma Systems
+ *
+ * af.cpp - RkISP1 AF hill climbing based control algorithm
+ */
+
+#include "af.h"
+
+/**
+ * \file af.h
+ */
+
+namespace libcamera {
+
+namespace ipa::rkisp1::algorithms {
+
+/**
+ * \class Af
+ * \brief AF contrast based hill climbing control algorithm for RkISP platforms
+ *
+ * Auto focus algorithm for RkISP platforms, based on the common hill climbing
+ * auto focus implementation
+ * (libcamera::ipa::algorithms::AfHillClimbing).
+ *
+ * This is the platform layer of the algorithm.
+ *
+ * Tuning file parameters:
+ * - **wait-frames-lens:** Number of frames that should be skipped when lens
+ *   position is changed. Lens movement takes some time and statistics measured
+ *   during the lens movement are unstable. Currently there is no way to know
+ *   when lens movement finished and this is a workaround for this. Wait a fixed
+ *   amount of time on each movement. This parameter should be set according
+ *   to the worst case  - the number of frames it takes to move lens between
+ *   limit positions.
+ * .
+ * \sa libcamera::ipa::algorithms::AfHillClimbing for additional tuning
+ * parameters.
+ *
+ * \todo Model the lens delay as number of frames required for the lens position
+ *   to stabilize in the CameraLens class.
+ */
+
+LOG_DEFINE_CATEGORY(RkISP1Af)
+
+/**
+ * \copydoc libcamera::ipa::Algorithm::init
+ */
+int Af::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData)
+{
+	waitFramesLens_ = tuningData["wait-frames-lens"].get<uint32_t>(1);
+
+	LOG(RkISP1Af, Debug) << "waitFramesLens_: " << waitFramesLens_;
+
+	return af.init(tuningData);
+}
+
+/**
+ * \copydoc libcamera::ipa::Algorithm::configure
+ */
+int Af::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
+{
+	const auto &lensConfig = context.configuration.lens;
+
+	af.configure(lensConfig.minFocusPosition, lensConfig.maxFocusPosition,
+		     configInfo.outputSize);
+
+	/*
+	 * Lens position is unknown at the startup, so initialize
+	 * the current position to something out of range.
+	 */
+	context.activeState.af.lensPosition = lensConfig.maxFocusPosition + 1;
+
+	return 0;
+}
+
+/**
+ * \copydoc libcamera::ipa::Algorithm::queueRequest
+ */
+void Af::queueRequest([[maybe_unused]] IPAContext &context,
+		      [[maybe_unused]] const uint32_t frame,
+		      [[maybe_unused]] IPAFrameContext &frameContext,
+		      const ControlList &controls)
+{
+	af.queueRequest(controls);
+}
+
+/**
+ * \copydoc libcamera::ipa::Algorithm::process
+ */
+void Af::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
+		 [[maybe_unused]] IPAFrameContext &frameContext,
+		 const rkisp1_stat_buffer *stats,
+		 [[maybe_unused]] ControlList &metadata)
+{
+	uint32_t sharpness = stats->params.af.window[0].sum;
+	uint32_t luminance = stats->params.af.window[0].lum;
+
+	LOG(RkISP1Af, Debug)
+		<< "lensPosition: " << context.activeState.af.lensPosition
+		<< ", Sharpness: " << sharpness
+		<< ", Luminance: " << luminance;
+
+	int32_t newLensPosition = af.process(sharpness);
+
+	if (newLensPosition != context.activeState.af.lensPosition) {
+		context.activeState.af.lensPosition = newLensPosition;
+		context.activeState.af.applyLensCtrls = true;
+		af.setFramesToSkip(waitFramesLens_);
+	}
+}
+
+REGISTER_IPA_ALGORITHM(Af, "Af")
+
+} /* namespace ipa::rkisp1::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/rkisp1/algorithms/af.h b/src/ipa/rkisp1/algorithms/af.h
new file mode 100644
index 00000000..3ba66d38
--- /dev/null
+++ b/src/ipa/rkisp1/algorithms/af.h
@@ -0,0 +1,43 @@ 
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2023, Theobroma Systems
+ *
+ * af.h - RkISP1 AF hill climbing based control algorithm
+ */
+
+#pragma once
+
+#include <linux/rkisp1-config.h>
+
+#include "libipa/algorithms/af_hill_climbing.h"
+
+#include "algorithm.h"
+
+namespace libcamera {
+
+namespace ipa::rkisp1::algorithms {
+
+class Af : public Algorithm
+{
+public:
+	int init(IPAContext &context, const YamlObject &tuningData) override;
+	int configure(IPAContext &context,
+		      const IPACameraSensorInfo &configInfo) override;
+	void queueRequest(IPAContext &context, uint32_t frame,
+			  IPAFrameContext &frameContext,
+			  const ControlList &controls) override;
+	void process(IPAContext &context, uint32_t frame,
+		     IPAFrameContext &frameContext,
+		     const rkisp1_stat_buffer *stats,
+		     ControlList &metadata) override;
+
+private:
+	ipa::algorithms::AfHillClimbing af;
+
+	/* Wait number of frames after changing lens position */
+	uint32_t waitFramesLens_ = 0;
+};
+
+} /* namespace ipa::rkisp1::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build
index 93a48329..ab7e44f3 100644
--- a/src/ipa/rkisp1/algorithms/meson.build
+++ b/src/ipa/rkisp1/algorithms/meson.build
@@ -1,6 +1,7 @@ 
 # SPDX-License-Identifier: CC0-1.0
 
 rkisp1_ipa_algorithms = files([
+    'af.cpp',
     'agc.cpp',
     'awb.cpp',
     'blc.cpp',
diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
index 401c098f..5c5f80c7 100644
--- a/src/ipa/rkisp1/ipa_context.cpp
+++ b/src/ipa/rkisp1/ipa_context.cpp
@@ -134,6 +134,19 @@  namespace libcamera::ipa::rkisp1 {
  * member may be read by any algorithm, but shall only be written by its owner.
  */
 
+/**
+ * \var IPAActiveState::af
+ * \brief State for the Automatic Focus Control algorithm
+ *
+ * \var IPAActiveState::af.lensPosition
+ * \brief Lens position calculated by the AF algorithm
+ *
+ * \var IPAActiveState::af.applyLensCtrls
+ * \brief Whether the lens position should be applied
+ *
+ * If true, IPA should send new controls to the PH to set new lens position.
+ */
+
 /**
  * \var IPAActiveState::agc
  * \brief State for the Automatic Gain Control algorithm
diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
index bfb6e1b7..2c2eec3b 100644
--- a/src/ipa/rkisp1/ipa_context.h
+++ b/src/ipa/rkisp1/ipa_context.h
@@ -58,6 +58,11 @@  struct IPASessionConfiguration {
 };
 
 struct IPAActiveState {
+	struct {
+		int32_t lensPosition;
+		bool applyLensCtrls;
+	} af;
+
 	struct {
 		struct {
 			uint32_t exposure;