[libcamera-devel,v4,06/11] libcamera: ipa: ipu3: Add an IPA skeleton for the IPU3 pipeline

11168 diff mbox series

Add an empty IPA skeleton for the IPU3 pipeline. The skeleton IPA
handles the flow of parameter and statistic buffers but does not read or
write anything in the buffers. It also allows the IPA to set sensor
controls but does not implement any logic to set optimal values and
instead sets the V4L2 exposure and gain controls to max and keeps them
at that setting.

This IPA is meant as a base to allow the pipeline handler to be wired up
to an IPA. The image algorithms can then later be added to the IPA
independently from also having to add plumbing to the pipeline handler.

Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
---
* Changes since v1
- Updated commit message.
- Rename IPU3_IPA_EVENT_PARSE_STAT to IPU3_IPA_EVENT_PARSE_STAT.
- Sort and drop unused headers.
- Fix style issue in meson.build and add dependency on libatomic.
- Switch to MappedFrameBuffer interface.
- Add result of IPA configuration status.

* Changes since v2
- Include stdint.h
- s/std::max<uint32_t>/std::max/
- Erase iterator instead of id in unmapBuffers()

* Changes since v3
- Add a new interface to be able to process controls separate from
  filling the parameters buffer.
---
 include/libcamera/ipa/ipu3.h |  23 ++++
 src/ipa/ipu3/ipu3.cpp        | 242 +++++++++++++++++++++++++++++++++++
 src/ipa/ipu3/meson.build     |  21 +++
 src/ipa/meson.build          |   2 +-
 4 files changed, 287 insertions(+), 1 deletion(-)
 create mode 100644 include/libcamera/ipa/ipu3.h
 create mode 100644 src/ipa/ipu3/ipu3.cpp
 create mode 100644 src/ipa/ipu3/meson.build

Hi Niklas,

Thank you for the patch.

On Fri, Feb 05, 2021 at 12:26:08AM +0100, Niklas Söderlund wrote:
> Add an empty IPA skeleton for the IPU3 pipeline. The skeleton IPA
> handles the flow of parameter and statistic buffers but does not read or
> write anything in the buffers. It also allows the IPA to set sensor
> controls but does not implement any logic to set optimal values and
> instead sets the V4L2 exposure and gain controls to max and keeps them
> at that setting.
> 
> This IPA is meant as a base to allow the pipeline handler to be wired up
> to an IPA. The image algorithms can then later be added to the IPA
> independently from also having to add plumbing to the pipeline handler.
> 
> Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>

Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

> ---
> * Changes since v1
> - Updated commit message.
> - Rename IPU3_IPA_EVENT_PARSE_STAT to IPU3_IPA_EVENT_PARSE_STAT.
> - Sort and drop unused headers.
> - Fix style issue in meson.build and add dependency on libatomic.
> - Switch to MappedFrameBuffer interface.
> - Add result of IPA configuration status.
> 
> * Changes since v2
> - Include stdint.h
> - s/std::max<uint32_t>/std::max/
> - Erase iterator instead of id in unmapBuffers()
> 
> * Changes since v3
> - Add a new interface to be able to process controls separate from
>   filling the parameters buffer.
> ---
>  include/libcamera/ipa/ipu3.h |  23 ++++
>  src/ipa/ipu3/ipu3.cpp        | 242 +++++++++++++++++++++++++++++++++++
>  src/ipa/ipu3/meson.build     |  21 +++
>  src/ipa/meson.build          |   2 +-
>  4 files changed, 287 insertions(+), 1 deletion(-)
>  create mode 100644 include/libcamera/ipa/ipu3.h
>  create mode 100644 src/ipa/ipu3/ipu3.cpp
>  create mode 100644 src/ipa/ipu3/meson.build
> 
> diff --git a/include/libcamera/ipa/ipu3.h b/include/libcamera/ipa/ipu3.h
> new file mode 100644
> index 0000000000000000..cbaaef04417b701b
> --- /dev/null
> +++ b/include/libcamera/ipa/ipu3.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Google Inc.
> + *
> + * ipu3.h - Image Processing Algorithm interface for IPU3
> + */
> +#ifndef __LIBCAMERA_IPA_INTERFACE_IPU3_H__
> +#define __LIBCAMERA_IPA_INTERFACE_IPU3_H__
> +
> +#ifndef __DOXYGEN__
> +
> +enum IPU3Operations {
> +	IPU3_IPA_ACTION_SET_SENSOR_CONTROLS = 1,
> +	IPU3_IPA_ACTION_PARAM_FILLED = 2,
> +	IPU3_IPA_ACTION_METADATA_READY = 3,
> +	IPU3_IPA_EVENT_PROCESS_CONTROLS = 4,
> +	IPU3_IPA_EVENT_STAT_READY = 5,
> +	IPU3_IPA_EVENT_FILL_PARAMS = 6,
> +};
> +
> +#endif /* __DOXYGEN__ */
> +
> +#endif /* __LIBCAMERA_IPA_INTERFACE_IPU3_H__ */
> diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
> new file mode 100644
> index 0000000000000000..b11b03efa6ceb666
> --- /dev/null
> +++ b/src/ipa/ipu3/ipu3.cpp
> @@ -0,0 +1,242 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Google Inc.
> + *
> + * ipu3.cpp - IPU3 Image Processing Algorithms
> + */
> +
> +#include <libcamera/ipa/ipu3.h>
> +
> +#include <stdint.h>
> +#include <sys/mman.h>
> +
> +#include <linux/intel-ipu3.h>
> +#include <linux/v4l2-controls.h>
> +
> +#include <libcamera/buffer.h>
> +#include <libcamera/control_ids.h>
> +#include <libcamera/ipa/ipa_interface.h>
> +#include <libcamera/ipa/ipa_module_info.h>
> +#include <libcamera/request.h>
> +
> +#include <libipa/ipa_interface_wrapper.h>
> +
> +#include "libcamera/internal/buffer.h"
> +#include "libcamera/internal/log.h"
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(IPAIPU3)
> +
> +class IPAIPU3 : public IPAInterface
> +{
> +public:
> +	int init([[maybe_unused]] const IPASettings &settings) override
> +	{
> +		return 0;
> +	}
> +	int start([[maybe_unused]] const IPAOperationData &data,
> +		  [[maybe_unused]] IPAOperationData *result) override { return 0; }
> +	void stop() override {}
> +
> +	void configure(const CameraSensorInfo &info,
> +		       const std::map<unsigned int, IPAStream> &streamConfig,
> +		       const std::map<unsigned int, const ControlInfoMap &> &entityControls,
> +		       const IPAOperationData &ipaConfig,
> +		       IPAOperationData *response) override;
> +	void mapBuffers(const std::vector<IPABuffer> &buffers) override;
> +	void unmapBuffers(const std::vector<unsigned int> &ids) override;
> +	void processEvent(const IPAOperationData &event) override;
> +
> +private:
> +	void processControls(unsigned int frame, const ControlList &controls);
> +	void fillParams(unsigned int frame, ipu3_uapi_params *params);
> +	void parseStatistics(unsigned int frame,
> +			     const ipu3_uapi_stats_3a *stats);
> +
> +	void setControls(unsigned int frame);
> +
> +	std::map<unsigned int, MappedFrameBuffer> buffers_;
> +
> +	ControlInfoMap ctrls_;
> +
> +	/* Camera sensor controls. */
> +	uint32_t exposure_;
> +	uint32_t minExposure_;
> +	uint32_t maxExposure_;
> +	uint32_t gain_;
> +	uint32_t minGain_;
> +	uint32_t maxGain_;
> +};
> +
> +void IPAIPU3::configure([[maybe_unused]] const CameraSensorInfo &info,
> +			[[maybe_unused]] const std::map<unsigned int, IPAStream> &streamConfig,
> +			const std::map<unsigned int, const ControlInfoMap &> &entityControls,
> +			[[maybe_unused]] const IPAOperationData &ipaConfig,
> +			[[maybe_unused]] IPAOperationData *result)
> +{
> +	if (entityControls.empty())
> +		return;
> +
> +	ctrls_ = entityControls.at(0);
> +
> +	const auto itExp = ctrls_.find(V4L2_CID_EXPOSURE);
> +	if (itExp == ctrls_.end()) {
> +		LOG(IPAIPU3, Error) << "Can't find exposure control";
> +		return;
> +	}
> +
> +	const auto itGain = ctrls_.find(V4L2_CID_ANALOGUE_GAIN);
> +	if (itGain == ctrls_.end()) {
> +		LOG(IPAIPU3, Error) << "Can't find gain control";
> +		return;
> +	}
> +
> +	minExposure_ = std::max(itExp->second.min().get<int32_t>(), 1);
> +	maxExposure_ = itExp->second.max().get<int32_t>();
> +	exposure_ = maxExposure_;
> +
> +	minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);
> +	maxGain_ = itGain->second.max().get<int32_t>();
> +	gain_ = maxGain_;
> +
> +	setControls(0);
> +}
> +
> +void IPAIPU3::mapBuffers(const std::vector<IPABuffer> &buffers)
> +{
> +	for (const IPABuffer &buffer : buffers) {
> +		const FrameBuffer fb(buffer.planes);
> +		buffers_.emplace(buffer.id,
> +				 MappedFrameBuffer(&fb, PROT_READ | PROT_WRITE));
> +	}
> +}
> +
> +void IPAIPU3::unmapBuffers(const std::vector<unsigned int> &ids)
> +{
> +	for (unsigned int id : ids) {
> +		auto it = buffers_.find(id);
> +		if (it == buffers_.end())
> +			continue;
> +
> +		buffers_.erase(it);
> +	}
> +}
> +
> +void IPAIPU3::processEvent(const IPAOperationData &event)
> +{
> +	switch (event.operation) {
> +	case IPU3_IPA_EVENT_PROCESS_CONTROLS: {
> +		unsigned int frame = event.data[0];
> +		processControls(frame, event.controls[0]);
> +		break;
> +	}
> +	case IPU3_IPA_EVENT_STAT_READY: {
> +		unsigned int frame = event.data[0];
> +		unsigned int bufferId = event.data[1];
> +
> +		auto it = buffers_.find(bufferId);
> +		if (it == buffers_.end()) {
> +			LOG(IPAIPU3, Error) << "Could not find stats buffer!";
> +			return;
> +		}
> +
> +		Span<uint8_t> mem = it->second.maps()[0];
> +		const ipu3_uapi_stats_3a *stats =
> +			reinterpret_cast<ipu3_uapi_stats_3a *>(mem.data());
> +
> +		parseStatistics(frame, stats);
> +		break;
> +	}
> +	case IPU3_IPA_EVENT_FILL_PARAMS: {
> +		unsigned int frame = event.data[0];
> +		unsigned int bufferId = event.data[1];
> +
> +		auto it = buffers_.find(bufferId);
> +		if (it == buffers_.end()) {
> +			LOG(IPAIPU3, Error) << "Could not find param buffer!";
> +			return;
> +		}
> +
> +		Span<uint8_t> mem = it->second.maps()[0];
> +		ipu3_uapi_params *params =
> +			reinterpret_cast<ipu3_uapi_params *>(mem.data());
> +
> +		fillParams(frame, params);
> +		break;
> +	}
> +	default:
> +		LOG(IPAIPU3, Error) << "Unknown event " << event.operation;
> +		break;
> +	}
> +}
> +
> +void IPAIPU3::processControls([[maybe_unused]] unsigned int frame,
> +			      [[maybe_unused]] const ControlList &controls)
> +{
> +	/* \todo Start processing for 'frame' based on 'controls'. */
> +}
> +
> +void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)
> +{
> +	/* Prepare parameters buffer. */
> +	memset(params, 0, sizeof(*params));
> +
> +	/* \todo Fill in parameters buffer. */
> +
> +	IPAOperationData op;
> +	op.operation = IPU3_IPA_ACTION_PARAM_FILLED;
> +
> +	queueFrameAction.emit(frame, op);
> +
> +	/* \todo Calculate new values for exposure_ and gain_. */
> +	setControls(frame);
> +}
> +
> +void IPAIPU3::parseStatistics(unsigned int frame,
> +			      [[maybe_unused]] const ipu3_uapi_stats_3a *stats)
> +{
> +	ControlList ctrls(controls::controls);
> +
> +	/* \todo React to statistics and update internal state machine. */
> +	/* \todo Add meta-data information to ctrls. */
> +
> +	IPAOperationData op;
> +	op.operation = IPU3_IPA_ACTION_METADATA_READY;
> +	op.controls.push_back(ctrls);
> +
> +	queueFrameAction.emit(frame, op);
> +}
> +
> +void IPAIPU3::setControls(unsigned int frame)
> +{
> +	IPAOperationData op;
> +	op.operation = IPU3_IPA_ACTION_SET_SENSOR_CONTROLS;
> +
> +	ControlList ctrls(ctrls_);
> +	ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure_));
> +	ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain_));
> +	op.controls.push_back(ctrls);
> +
> +	queueFrameAction.emit(frame, op);
> +}
> +
> +/*
> + * External IPA module interface
> + */
> +
> +extern "C" {
> +const struct IPAModuleInfo ipaModuleInfo = {
> +	IPA_MODULE_API_VERSION,
> +	1,
> +	"PipelineHandlerIPU3",
> +	"ipu3",
> +};
> +
> +struct ipa_context *ipaCreate()
> +{
> +	return new IPAInterfaceWrapper(std::make_unique<IPAIPU3>());
> +}
> +}
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build
> new file mode 100644
> index 0000000000000000..444c82453eac42ff
> --- /dev/null
> +++ b/src/ipa/ipu3/meson.build
> @@ -0,0 +1,21 @@
> +# SPDX-License-Identifier: CC0-1.0
> +
> +ipa_name = 'ipa_ipu3'
> +
> +mod = shared_module(ipa_name,
> +                    'ipu3.cpp',
> +                    name_prefix : '',
> +                    include_directories : [ ipa_includes, libipa_includes ],
> +                    dependencies : [ libatomic, libcamera_dep ],
> +                    link_with : libipa,
> +                    install : true,
> +                    install_dir : ipa_install_dir)
> +
> +if ipa_sign_module
> +    custom_target(ipa_name + '.so.sign',
> +                  input : mod,
> +                  output : ipa_name + '.so.sign',
> +                  command : [ ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@' ],
> +                  install : false,
> +                  build_by_default : true)
> +endif
> diff --git a/src/ipa/meson.build b/src/ipa/meson.build
> index 5a5de267c1477d24..9d623f227a1f9feb 100644
> --- a/src/ipa/meson.build
> +++ b/src/ipa/meson.build
> @@ -19,7 +19,7 @@ subdir('libipa')
>  
>  ipa_sign = files('ipa-sign.sh')
>  
> -ipas = ['raspberrypi', 'rkisp1', 'vimc']
> +ipas = ['ipu3', 'raspberrypi', 'rkisp1', 'vimc']
>  ipa_names = []
>  
>  foreach pipeline : get_option('pipelines')