From patchwork Thu Dec 12 02:12:45 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Keke Li X-Patchwork-Id: 22292 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id B1087C3260 for ; Thu, 12 Dec 2024 02:12:56 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 943DA67EBE; Thu, 12 Dec 2024 03:12:53 +0100 (CET) Received: from mail-sh.amlogic.com (unknown [58.32.228.46]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 5A05667EB1 for ; Thu, 12 Dec 2024 03:12:51 +0100 (CET) Received: from droid10.amlogic.com (10.18.11.213) by mail-sh.amlogic.com (10.18.11.5) with Microsoft SMTP Server id 15.1.2507.39; Thu, 12 Dec 2024 10:12:50 +0800 From: Keke Li To: CC: Keke Li Subject: [PATCH 1/2] libcamera: pipeline: Add C3 ISP pipeline handler Date: Thu, 12 Dec 2024 10:12:45 +0800 Message-ID: <20241212021246.115736-2-keke.li@amlogic.com> X-Mailer: git-send-email 2.29.0 In-Reply-To: <20241212021246.115736-1-keke.li@amlogic.com> References: <20241212021246.115736-1-keke.li@amlogic.com> MIME-Version: 1.0 X-Originating-IP: [10.18.11.213] X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The Amlogic C3 ISP pipeline handler supports 3-channel image output, 1-channel 3A statistical information ouput and 1-channel parameters input. Signed-off-by: Keke Li --- include/libcamera/ipa/c3isp.mojom | 42 + include/libcamera/ipa/meson.build | 2 + include/linux/videodev2.h | 4 + meson_options.txt | 1 + src/libcamera/pipeline/c3-isp/c3-isp.cpp | 1161 +++++++++++++++++++++ src/libcamera/pipeline/c3-isp/meson.build | 5 + 6 files changed, 1215 insertions(+) create mode 100644 include/libcamera/ipa/c3isp.mojom create mode 100644 src/libcamera/pipeline/c3-isp/c3-isp.cpp create mode 100644 src/libcamera/pipeline/c3-isp/meson.build diff --git a/include/libcamera/ipa/c3isp.mojom b/include/libcamera/ipa/c3isp.mojom new file mode 100644 index 00000000..e41fb243 --- /dev/null +++ b/include/libcamera/ipa/c3isp.mojom @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +/* + * \todo Document the interface and remove the related EXCLUDE_PATTERNS entry. + */ + +module ipa.c3isp; + +import "include/libcamera/ipa/core.mojom"; + +struct IPAConfigInfo { + libcamera.IPACameraSensorInfo sensorInfo; + libcamera.ControlInfoMap sensorControls; + uint32 paramFormat; +}; + +interface IPAC3ISPInterface { + init(libcamera.IPASettings settings, + uint32 hwRevision, + libcamera.IPACameraSensorInfo sensorInfo, + libcamera.ControlInfoMap sensorControls) + => (int32 ret, libcamera.ControlInfoMap ipaControls); + start() => (int32 ret); + stop(); + + configure(IPAConfigInfo configInfo) + => (int32 ret, libcamera.ControlInfoMap ipaControls); + + mapBuffers(array buffers); + unmapBuffers(array ids); + + [async] queueRequest(uint32 frame, libcamera.ControlList reqControls); + [async] fillParamsBuffer(uint32 frame, uint32 bufferId); + [async] processStatsBuffer(uint32 frame, uint32 bufferId, + libcamera.ControlList sensorControls); +}; + +interface IPAC3ISPEventInterface { + paramsBufferReady(uint32 frame, uint32 bytesused); + setSensorControls(uint32 frame, libcamera.ControlList sensorControls); + metadataReady(uint32 frame, libcamera.ControlList metadata); +}; diff --git a/include/libcamera/ipa/meson.build b/include/libcamera/ipa/meson.build index bf55e124..9e47842b 100644 --- a/include/libcamera/ipa/meson.build +++ b/include/libcamera/ipa/meson.build @@ -63,6 +63,7 @@ libcamera_ipa_headers += custom_target('core_ipa_serializer_h', # Mapping from pipeline handler name to mojom file pipeline_ipa_mojom_mapping = { + 'c3-isp': 'c3isp.mojom', 'ipu3': 'ipu3.mojom', 'rkisp1': 'rkisp1.mojom', 'rpi/vc4': 'raspberrypi.mojom', @@ -70,6 +71,7 @@ pipeline_ipa_mojom_mapping = { 'vimc': 'vimc.mojom', } + # # Generate headers from templates. # diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index 3829c0b6..d5bbed29 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -831,6 +831,10 @@ struct v4l2_pix_format { #define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A Statistics */ #define V4L2_META_FMT_RK_ISP1_EXT_PARAMS v4l2_fourcc('R', 'K', '1', 'E') /* Rockchip ISP1 3a Extensible Parameters */ +/* Vendor specific - used for C3 ISP */ +#define V4L2_META_FMT_C3ISP_PARAMS v4l2_fourcc('C', 'P', 'R', 'M') /* Amlogic C3 ISP Parameters */ +#define V4L2_META_FMT_C3ISP_STATS v4l2_fourcc('C', 'S', 'T', 'S') /* Amlogic C3 ISP Statistics */ + /* Vendor specific - used for RaspberryPi PiSP */ #define V4L2_META_FMT_RPI_BE_CFG v4l2_fourcc('R', 'P', 'B', 'C') /* PiSP BE configuration */ diff --git a/meson_options.txt b/meson_options.txt index c91cd241..bd782830 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -46,6 +46,7 @@ option('pipelines', choices : [ 'all', 'auto', + 'c3-isp', 'imx8-isi', 'ipu3', 'mali-c55', diff --git a/src/libcamera/pipeline/c3-isp/c3-isp.cpp b/src/libcamera/pipeline/c3-isp/c3-isp.cpp new file mode 100644 index 00000000..524027d4 --- /dev/null +++ b/src/libcamera/pipeline/c3-isp/c3-isp.cpp @@ -0,0 +1,1161 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * Pipeline Handler for Amlogic C3 ISP + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/delayed_controls.h" +#include "libcamera/internal/device_enumerator.h" +#include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/ipa_manager.h" +#include "libcamera/internal/media_device.h" +#include "libcamera/internal/pipeline_handler.h" +#include "libcamera/internal/v4l2_subdevice.h" +#include "libcamera/internal/v4l2_videodevice.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(C3ISP) + +class PipelineHandlerC3ISP; +class C3ISPCameraData; + +const std::map C3ISPFmtToCode = { + { formats::R8, MEDIA_BUS_FMT_YUV8_1X24 }, + { formats::NV12, MEDIA_BUS_FMT_YUV8_1X24 }, + { formats::NV21, MEDIA_BUS_FMT_YUV8_1X24 }, + { formats::NV16, MEDIA_BUS_FMT_YUV8_1X24 }, + { formats::NV61, MEDIA_BUS_FMT_YUV8_1X24 }, +}; + +constexpr Size kC3ISPMinSize = { 160, 120 }; +constexpr Size kC3ISPMaxSize = { 2888, 2240 }; + +struct C3ISPFrameInfo { + unsigned int frame; + Request *request; + + FrameBuffer *paramBuffer; + FrameBuffer *statBuffer; + FrameBuffer *viewBuffer; + FrameBuffer *stillBuffer; + FrameBuffer *videoBuffer; + + bool paramDequeued; + bool metadataProcessed; +}; + +class C3ISPFrames +{ +public: + C3ISPFrames(PipelineHandler *pipe); + + C3ISPFrameInfo *create(const C3ISPCameraData *data, Request *request); + int destroy(unsigned int frame); + void clear(); + + C3ISPFrameInfo *find(unsigned int frame); + C3ISPFrameInfo *find(FrameBuffer *buffer); + C3ISPFrameInfo *find(Request *request); + +private: + PipelineHandlerC3ISP *pipe_; + std::map frameInfo_; +}; + +class C3ISPCameraData : public Camera::Private +{ +public: + C3ISPCameraData(PipelineHandler *pipe, MediaEntity *entity) + : Camera::Private(pipe), entity_(entity), frame_(0), frameInfo_(pipe) + { + } + + int init(); + PipelineHandlerC3ISP *pipe(); + int loadIPA(unsigned int hwRevision); + + MediaEntity *entity_; + std::unique_ptr sensor_; + std::unique_ptr delayedCtrls_; + std::unique_ptr csi_; + std::unique_ptr adap_; + Stream viewStream; + Stream stillStream; + Stream videoStream; + unsigned int frame_; + std::vector ipaBuffers_; + C3ISPFrames frameInfo_; + + std::unique_ptr ipa_; + +private: + void paramFilled(unsigned int frame, unsigned int bytesused); + void setSensorControls(unsigned int frame, + const ControlList &sensorControls); + void metadataReady(unsigned int frame, const ControlList &metadata); +}; + +class C3ISPCameraConfiguration : public CameraConfiguration +{ +public: + C3ISPCameraConfiguration(C3ISPCameraData *data) + : CameraConfiguration(), data_(data) + { + } + + Status validate() override; + + V4L2SubdeviceFormat sensorFormat_; + +private: + static constexpr unsigned int kMaxStreams = 3; + + const C3ISPCameraData *data_; +}; + +class PipelineHandlerC3ISP : public PipelineHandler +{ +public: + PipelineHandlerC3ISP(CameraManager *manager); + + std::unique_ptr generateConfiguration(Camera *camera, + Span roles) override; + int configure(Camera *camera, CameraConfiguration *config) override; + + int exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) override; + + int start(Camera *camera, const ControlList *controls) override; + void stopDevice(Camera *camera) override; + + int queueRequestDevice(Camera *camera, Request *request) override; + + void bufferReady(FrameBuffer *buffer); + void statReady(FrameBuffer *buffer); + void paramReady(FrameBuffer *buffer); + + bool match(DeviceEnumerator *enumerator) override; + +private: + friend C3ISPCameraData; + friend C3ISPFrames; + + struct C3ISPPipe { + std::unique_ptr resizer; + std::unique_ptr cap; + Stream *stream; + }; + + enum { + C3ISPVIEW, + C3ISPSTILL, + C3ISPVIDEO, + C3ISPNumPipes, + }; + + C3ISPCameraData *cameraData(Camera *camera) + { + return static_cast(camera->_d()); + } + + C3ISPPipe *pipeFromStream(C3ISPCameraData *data, Stream *stream) + { + if (stream == &data->viewStream) + return &pipes_[C3ISPVIEW]; + else if (stream == &data->stillStream) + return &pipes_[C3ISPSTILL]; + else if (stream == &data->videoStream) + return &pipes_[C3ISPVIDEO]; + else + LOG(C3ISP, Fatal) << "Invalid stream: " << stream; + + return nullptr; + } + + C3ISPPipe *pipeFromStream(C3ISPCameraData *data, const Stream *stream) + { + return pipeFromStream(data, const_cast(stream)); + } + + void resetPipes() + { + for (C3ISPPipe &pipe : pipes_) + pipe.stream = nullptr; + } + + int pipeStart(); + void pipeStop(); + + int setConfigStreams(CameraConfiguration *config); + int configureProcessedStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat); + + bool createCamera(MediaLink *ispLink); + void tryCompleteRequest(C3ISPFrameInfo *info); + int allocateBuffers(Camera *camera); + int freeBuffers(Camera *camera); + + MediaDevice *media_; + std::unique_ptr isp_; + std::unique_ptr param_; + std::unique_ptr stat_; + + std::vector> paramBuffers_; + std::vector> statBuffers_; + std::queue availableParamBuffers_; + std::queue availableStatBuffers_; + + std::vector streams_; + std::array pipes_; + + Camera *activeCamera_; +}; + +C3ISPFrames::C3ISPFrames(PipelineHandler *pipe) + : pipe_(static_cast(pipe)) +{ +} + +C3ISPFrameInfo *C3ISPFrames::create(const C3ISPCameraData *data, Request *request) +{ + unsigned int frame = data->frame_; + + FrameBuffer *paramBuffer = nullptr; + FrameBuffer *statBuffer = nullptr; + + if (pipe_->availableParamBuffers_.empty()) { + LOG(C3ISP, Error) << "Param buffer queue empty"; + return nullptr; + } + + if (pipe_->availableStatBuffers_.empty()) { + LOG(C3ISP, Error) << "Stat buffer queue empty"; + return nullptr; + } + + paramBuffer = pipe_->availableParamBuffers_.front(); + pipe_->availableParamBuffers_.pop(); + + statBuffer = pipe_->availableStatBuffers_.front(); + pipe_->availableStatBuffers_.pop(); + + FrameBuffer *viewBuffer = request->findBuffer(&data->viewStream); + FrameBuffer *stillBuffer = request->findBuffer(&data->stillStream); + FrameBuffer *videoBuffer = request->findBuffer(&data->videoStream); + + C3ISPFrameInfo *info = new C3ISPFrameInfo; + + info->frame = frame; + info->request = request; + info->paramBuffer = paramBuffer; + info->statBuffer = statBuffer; + info->viewBuffer = viewBuffer; + info->stillBuffer = stillBuffer; + info->videoBuffer = videoBuffer; + info->paramDequeued = false; + info->metadataProcessed = false; + + frameInfo_[frame] = info; + + return info; +} + +int C3ISPFrames::destroy(unsigned int frame) +{ + C3ISPFrameInfo *info = find(frame); + if (!info) + return -ENOENT; + + pipe_->availableParamBuffers_.push(info->paramBuffer); + pipe_->availableStatBuffers_.push(info->statBuffer); + + frameInfo_.erase(info->frame); + + delete info; + + return 0; +} + +void C3ISPFrames::clear() +{ + for (const auto &entry : frameInfo_) { + C3ISPFrameInfo *info = entry.second; + + pipe_->availableParamBuffers_.push(info->paramBuffer); + pipe_->availableStatBuffers_.push(info->statBuffer); + + delete info; + } + + frameInfo_.clear(); +} + +C3ISPFrameInfo *C3ISPFrames::find(unsigned int frame) +{ + auto itInfo = frameInfo_.find(frame); + + if (itInfo != frameInfo_.end()) + return itInfo->second; + + LOG(C3ISP, Fatal) << "Can't locate info from frame"; + + return nullptr; +} + +C3ISPFrameInfo *C3ISPFrames::find(FrameBuffer *buffer) +{ + for (auto &itInfo : frameInfo_) { + C3ISPFrameInfo *info = itInfo.second; + + if (info->paramBuffer == buffer || + info->statBuffer == buffer || + info->viewBuffer == buffer || + info->stillBuffer == buffer || + info->videoBuffer == buffer) + return info; + } + + LOG(C3ISP, Fatal) << "Can't locate info from buffer"; + + return nullptr; +} + +C3ISPFrameInfo *C3ISPFrames::find(Request *request) +{ + for (auto &itInfo : frameInfo_) { + C3ISPFrameInfo *info = itInfo.second; + + if (info->request == request) + return info; + } + + LOG(C3ISP, Fatal) << "Can't locate info from request"; + + return nullptr; +} + +int C3ISPCameraData::init() +{ + int ret; + + /* Register a CameraSensor */ + sensor_ = CameraSensorFactoryBase::create(entity_); + if (!sensor_) + return ret; + + const MediaPad *sensorSrc = entity_->getPadByIndex(0); + MediaEntity *csiEntity = sensorSrc->links()[0]->sink()->entity(); + + csi_ = std::make_unique(csiEntity); + if (csi_->open()) { + LOG(C3ISP, Error) << "Failed to open CSI-2 subdevice"; + return false; + } + + const MediaPad *csiSrc = csiEntity->getPadByIndex(1); + MediaEntity *adapEntity = csiSrc->links()[0]->sink()->entity(); + + adap_ = std::make_unique(adapEntity); + if (adap_->open()) { + LOG(C3ISP, Error) << "Failed to open adapter subdevice"; + return false; + } + + return 0; +} + +PipelineHandlerC3ISP *C3ISPCameraData::pipe() +{ + return static_cast(Camera::Private::pipe()); +} + +int C3ISPCameraData::loadIPA(unsigned int hwRevision) +{ + ipa_ = IPAManager::createIPA(pipe(), 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->setSensorControls.connect(this, &C3ISPCameraData::setSensorControls); + ipa_->paramsBufferReady.connect(this, &C3ISPCameraData::paramFilled); + ipa_->metadataReady.connect(this, &C3ISPCameraData::metadataReady); + + /* + * The API tuning file is made from the sensor name unless the + * environment variable overrides it. + */ + std::string ipaTuningFile; + char const *configFromEnv = utils::secure_getenv("LIBCAMERA_C3ISP_TUNING_FILE"); + if (!configFromEnv || *configFromEnv == '\0') { + ipaTuningFile = ipa_->configurationFile(sensor_->model() + ".yaml"); + if (ipaTuningFile.empty()) + ipaTuningFile = ipa_->configurationFile("uncalibrated.yaml"); + } else { + ipaTuningFile = std::string(configFromEnv); + } + + IPACameraSensorInfo sensorInfo{}; + int ret = sensor_->sensorInfo(&sensorInfo); + if (ret) { + LOG(C3ISP, Error) << "Invalid semsor information"; + return ret; + } + + ret = ipa_->init({ ipaTuningFile, sensor_->model() }, hwRevision, + sensorInfo, sensor_->controls(), &controlInfo_); + if (ret) { + LOG(C3ISP, Error) << "Failed to initialize IPA"; + return ret; + } + + return 0; +} + +void C3ISPCameraData::paramFilled(unsigned int frame, unsigned int bytesused) +{ + PipelineHandlerC3ISP *pipe = C3ISPCameraData::pipe(); + C3ISPFrameInfo *info = frameInfo_.find(frame); + if (!info) + return; + + info->paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused; + pipe->param_->queueBuffer(info->paramBuffer); + pipe->stat_->queueBuffer(info->statBuffer); + + if (info->viewBuffer) + pipe->pipes_[PipelineHandlerC3ISP::C3ISPVIEW].cap->queueBuffer(info->viewBuffer); + + if (info->stillBuffer) + pipe->pipes_[PipelineHandlerC3ISP::C3ISPSTILL].cap->queueBuffer(info->stillBuffer); + + if (info->videoBuffer) + pipe->pipes_[PipelineHandlerC3ISP::C3ISPVIDEO].cap->queueBuffer(info->videoBuffer); +} + +void C3ISPCameraData::setSensorControls([[maybe_unused]] unsigned int frame, + const ControlList &sensorControls) +{ + delayedCtrls_->push(sensorControls); +} + +void C3ISPCameraData::metadataReady(unsigned int frame, const ControlList &metadata) +{ + C3ISPFrameInfo *info = frameInfo_.find(frame); + if (!info) + return; + + info->request->metadata().merge(metadata); + info->metadataProcessed = true; + + pipe()->tryCompleteRequest(info); +} + +namespace { + +const std::map rawFormats = { + { formats::SBGGR10, MEDIA_BUS_FMT_SBGGR10_1X10 }, + { formats::SGBRG10, MEDIA_BUS_FMT_SGBRG10_1X10 }, + { formats::SGRBG10, MEDIA_BUS_FMT_SGRBG10_1X10 }, + { formats::SRGGB10, MEDIA_BUS_FMT_SRGGB10_1X10 }, + { formats::SBGGR12, MEDIA_BUS_FMT_SBGGR12_1X12 }, + { formats::SGBRG12, MEDIA_BUS_FMT_SGBRG12_1X12 }, + { formats::SGRBG12, MEDIA_BUS_FMT_SGRBG12_1X12 }, + { formats::SRGGB12, MEDIA_BUS_FMT_SRGGB12_1X12 }, +}; + +}; + +CameraConfiguration::Status C3ISPCameraConfiguration::validate() +{ + const CameraSensor *sensor = data_->sensor_.get(); + Status status = Valid; + + if (config_.empty()) + return Invalid; + + if (config_.size() > kMaxStreams) { + config_.resize(kMaxStreams); + status = Adjusted; + } + + Size maxSize; + + for (StreamConfiguration &config : config_) { + const auto it = C3ISPFmtToCode.find(config.pixelFormat); + if (it == C3ISPFmtToCode.end()) { + LOG(C3ISP, Debug) + << "Format adjusted to " << formats::NV12; + config.pixelFormat = formats::NV12; + status = Adjusted; + } + + Size size = std::clamp(config.size, kC3ISPMinSize, kC3ISPMaxSize); + if (size != config.size) { + LOG(C3ISP, Debug) + << "Size adjusted to " << size; + config.size = size; + status = Adjusted; + } + + maxSize = std::max(maxSize, config.size); + } + + std::vector mbusCodes; + + std::transform(rawFormats.begin(), rawFormats.end(), + std::back_inserter(mbusCodes), + [](const auto &value) { return value.second; }); + + sensorFormat_ = sensor->getFormat(mbusCodes, maxSize); + + if (sensorFormat_.size.isNull()) + sensorFormat_.size = sensor->resolution(); + + return status; +} + +PipelineHandlerC3ISP::PipelineHandlerC3ISP(CameraManager *manager) + : PipelineHandler(manager) +{ +} + +std::unique_ptr +PipelineHandlerC3ISP::generateConfiguration(Camera *camera, + Span roles) +{ + C3ISPCameraData *data = cameraData(camera); + std::unique_ptr config = + std::make_unique(data); + + if (roles.empty()) + return config; + + streams_.clear(); + + for (const StreamRole &role : roles) { + PixelFormat pixelFormat; + Size size = std::min(Size{ 1920, 1080 }, data->sensor_->resolution()); + + switch (role) { + case StreamRole::StillCapture: + pixelFormat = formats::NV12; + streams_.push_back(&data->stillStream); + break; + + case StreamRole::VideoRecording: + pixelFormat = formats::NV12; + streams_.push_back(&data->videoStream); + break; + + case StreamRole::Viewfinder: + pixelFormat = formats::NV12; + streams_.push_back(&data->viewStream); + break; + + default: + LOG(C3ISP, Error) << "Invalid stream role: " << role; + return nullptr; + } + + std::map> formats; + for (const auto &c3Format : C3ISPFmtToCode) { + PixelFormat pixFmt = c3Format.first; + Size maxSize = std::min(kC3ISPMaxSize, data->sensor_->resolution()); + formats[pixFmt] = { kC3ISPMinSize, maxSize }; + } + + StreamFormats streamFormats(formats); + StreamConfiguration cfg(streamFormats); + cfg.pixelFormat = pixelFormat; + cfg.bufferCount = 4; + cfg.size = size; + + config->addConfiguration(cfg); + } + + if (config->validate() == CameraConfiguration::Invalid) + return nullptr; + + return config; +} + +int PipelineHandlerC3ISP::configureProcessedStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat) +{ + Stream *stream = config.stream(); + C3ISPPipe *pipe = pipeFromStream(data, stream); + V4L2SubdeviceFormat rszFormat; + + const MediaEntity *resizerEntity = pipe->resizer->entity(); + int ret = resizerEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + ret = resizerEntity->getPadByIndex(1)->links()[0]->setEnabled(true); + if (ret) + return ret; + + rszFormat.code = C3ISPFmtToCode.find(config.pixelFormat)->second; + rszFormat.size = subdevFormat.size; + rszFormat.colorSpace = subdevFormat.colorSpace; + + ret = pipe->resizer->setFormat(0, &rszFormat); + if (ret) + return ret; + + Rectangle cropRect = { 0, 0, rszFormat.size }; + ret = pipe->resizer->setSelection(0, V4L2_SEL_TGT_CROP, &cropRect); + if (ret) + return ret; + + Rectangle scaleRect = { 0, 0, config.size }; + ret = pipe->resizer->setSelection(0, V4L2_SEL_TGT_COMPOSE, &scaleRect); + if (ret) + return ret; + + return 0; +} + +int PipelineHandlerC3ISP::setConfigStreams(CameraConfiguration *config) +{ + if (config->size() != streams_.size()) { + LOG(C3ISP, Error) << "Invalid configuration size: " << config->size(); + return -EINVAL; + } + + for (unsigned int i = 0; i < config->size(); i++) + config->at(i).setStream(streams_[i]); + + return 0; +} + +int PipelineHandlerC3ISP::configure(Camera *camera, CameraConfiguration *config) +{ + resetPipes(); + + int ret = media_->disableLinks(); + if (ret) + return ret; + + /* + * The stream has been set to nullptr in Camera::configure, + * so need to set stream. + */ + ret = setConfigStreams(config); + if (ret) + return ret; + + /* Link the graph */ + C3ISPCameraData *data = cameraData(camera); + + const MediaEntity *csiEntity = data->csi_->entity(); + ret = csiEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + const MediaEntity *adapEntity = data->adap_->entity(); + ret = adapEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + const MediaEntity *ispEntity = isp_->entity(); + ret = ispEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + ret = ispEntity->getPadByIndex(1)->links()[0]->setEnabled(true); + if (ret) + return ret; + + ret = ispEntity->getPadByIndex(2)->links()[0]->setEnabled(true); + if (ret) + return ret; + + C3ISPCameraConfiguration *c3Config = static_cast(config); + V4L2SubdeviceFormat subdevFormat = c3Config->sensorFormat_; + + ret = data->sensor_->setFormat(&subdevFormat); + if (ret) + return ret; + + ret = data->csi_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + ret = data->adap_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + ret = isp_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + V4L2SubdeviceFormat ispSrcVideoFormat = subdevFormat; + ispSrcVideoFormat.code = MEDIA_BUS_FMT_YUV8_1X24; + ret = isp_->setFormat(3, &ispSrcVideoFormat); + if (ret) + return ret; + + V4L2DeviceFormat paramFormat; + paramFormat.fourcc = V4L2PixelFormat(V4L2_META_FMT_C3ISP_PARAMS); + ret = param_->setFormat(¶mFormat); + if (ret) + return ret; + + V4L2DeviceFormat statFormat; + statFormat.fourcc = V4L2PixelFormat(V4L2_META_FMT_C3ISP_STATS); + ret = stat_->setFormat(&statFormat); + if (ret) + return ret; + + for (const StreamConfiguration &streamConfig : *config) { + Stream *stream = streamConfig.stream(); + C3ISPPipe *pipe = pipeFromStream(data, stream); + + ret = configureProcessedStream(data, streamConfig, subdevFormat); + if (ret) { + LOG(C3ISP, Error) << "Failed to configure process stream"; + return ret; + } + + V4L2DeviceFormat captureFormat; + captureFormat.fourcc = pipe->cap->toV4L2PixelFormat(streamConfig.pixelFormat); + captureFormat.size = streamConfig.size; + + ret = pipe->cap->setFormat(&captureFormat); + if (ret) + return ret; + + pipe->stream = stream; + } + + /* Configure IPA module */ + ipa::c3isp::IPAConfigInfo ipaConfig{}; + + ret = data->sensor_->sensorInfo(&ipaConfig.sensorInfo); + if (ret) + return ret; + + ipaConfig.sensorControls = data->sensor_->controls(); + + ret = data->ipa_->configure(ipaConfig, &data->controlInfo_); + if (ret) { + LOG(C3ISP, Error) << "Failed to configure IPA"; + return ret; + } + + return 0; +} + +int PipelineHandlerC3ISP::exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) +{ + C3ISPPipe *pipe = pipeFromStream(cameraData(camera), stream); + unsigned int count = stream->configuration().bufferCount; + + return pipe->cap->exportBuffers(count, buffers); +} + +int PipelineHandlerC3ISP::allocateBuffers(Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + unsigned int ipaBufferId = 1; + + int ret = param_->allocateBuffers(4, ¶mBuffers_); + if (ret < 0) + return ret; + + ret = stat_->allocateBuffers(4, &statBuffers_); + if (ret < 0) { + paramBuffers_.clear(); + statBuffers_.clear(); + return ret; + } + + for (std::unique_ptr &buffer : paramBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); + availableParamBuffers_.push(buffer.get()); + } + + for (std::unique_ptr &buffer : statBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); + availableStatBuffers_.push(buffer.get()); + } + + data->ipa_->mapBuffers(data->ipaBuffers_); + + return 0; +} + +int PipelineHandlerC3ISP::freeBuffers(Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + + while (!availableStatBuffers_.empty()) + availableStatBuffers_.pop(); + + while (!availableParamBuffers_.empty()) + availableParamBuffers_.pop(); + + paramBuffers_.clear(); + statBuffers_.clear(); + + std::vector ids; + for (IPABuffer &ipabuf : data->ipaBuffers_) + ids.push_back(ipabuf.id); + + data->ipa_->unmapBuffers(ids); + data->ipaBuffers_.clear(); + + if (param_->releaseBuffers()) + LOG(C3ISP, Error) << "Failed to release param buffers"; + + if (stat_->releaseBuffers()) + LOG(C3ISP, Error) << "Failed to release stat buffers"; + + return 0; +} + +int PipelineHandlerC3ISP::pipeStart() +{ + for (C3ISPPipe &pipe : pipes_) { + if (!pipe.stream) + continue; + + Stream *stream = pipe.stream; + + int ret = pipe.cap->importBuffers(stream->configuration().bufferCount); + if (ret) { + LOG(C3ISP, Error) << "Failed to import buffers"; + return ret; + } + + ret = pipe.cap->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start stream"; + return ret; + } + } + + return 0; +} + +void PipelineHandlerC3ISP::pipeStop() +{ + for (C3ISPPipe &pipe : pipes_) { + if (!pipe.stream) + continue; + + pipe.cap->streamOff(); + pipe.cap->releaseBuffers(); + } +} + +int PipelineHandlerC3ISP::start([[maybe_unused]] Camera *camera, + [[maybe_unused]] const ControlList *controls) +{ + C3ISPCameraData *data = cameraData(camera); + int ret; + + ret = allocateBuffers(camera); + if (ret < 0) + return ret; + + ret = data->ipa_->start(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start IPA"; + goto error; + } + + data->frame_ = 0; + + ret = param_->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start param"; + goto error; + } + + ret = stat_->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start stat"; + goto error; + } + + ret = pipeStart(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start pipe"; + goto error; + } + + ret = isp_->setFrameStartEnabled(true); + if (ret) { + LOG(C3ISP, Error) << "Failed to set frame start"; + goto error; + } + + activeCamera_ = camera; + + return 0; + +error: + pipeStop(); + stat_->streamOff(); + param_->streamOff(); + data->ipa_->stop(); + freeBuffers(camera); + LOG(C3ISP, Error) << "Failed to start camera " << camera->id(); + + return ret; +} + +void PipelineHandlerC3ISP::stopDevice([[maybe_unused]] Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + + isp_->setFrameStartEnabled(false); + + data->ipa_->stop(); + + pipeStop(); + + stat_->streamOff(); + param_->streamOff(); + + data->frameInfo_.clear(); + + freeBuffers(camera); + + activeCamera_ = nullptr; +} + +int PipelineHandlerC3ISP::queueRequestDevice(Camera *camera, Request *request) +{ + C3ISPCameraData *data = cameraData(camera); + + C3ISPFrameInfo *info = data->frameInfo_.create(data, request); + if (!info) + return -ENOENT; + + data->ipa_->queueRequest(data->frame_, request->controls()); + + data->ipa_->fillParamsBuffer(data->frame_, info->paramBuffer->cookie()); + + data->frame_++; + + return 0; +} + +void PipelineHandlerC3ISP::tryCompleteRequest(C3ISPFrameInfo *info) +{ + C3ISPCameraData *data = cameraData(activeCamera_); + Request *request = info->request; + + if (request->hasPendingBuffers()) + return; + + if (!info->metadataProcessed) + return; + + if (!info->paramDequeued) + return; + + data->frameInfo_.destroy(info->frame); + + completeRequest(request); +} + +void PipelineHandlerC3ISP::bufferReady(FrameBuffer *buffer) +{ + C3ISPCameraData *data = cameraData(activeCamera_); + + C3ISPFrameInfo *info = data->frameInfo_.find(buffer); + if (!info) + return; + + const FrameMetadata &metadata = buffer->metadata(); + Request *request = buffer->request(); + + if (metadata.status != FrameMetadata::FrameCancelled) { + request->metadata().set(controls::SensorTimestamp, + metadata.timestamp); + } + + completeBuffer(request, buffer); + tryCompleteRequest(info); +} + +void PipelineHandlerC3ISP::statReady(FrameBuffer *buffer) +{ + C3ISPCameraData *data = cameraData(activeCamera_); + + C3ISPFrameInfo *info = data->frameInfo_.find(buffer); + if (!info) + return; + + if (buffer->metadata().status == FrameMetadata::FrameCancelled) { + info->metadataProcessed = true; + tryCompleteRequest(info); + return; + } + + if (data->frame_ <= buffer->metadata().sequence) + data->frame_ = buffer->metadata().sequence + 1; + + data->ipa_->processStatsBuffer(info->frame, info->statBuffer->cookie(), + data->delayedCtrls_->get(buffer->metadata().sequence)); +} + +void PipelineHandlerC3ISP::paramReady(FrameBuffer *buffer) +{ + C3ISPCameraData *data = cameraData(activeCamera_); + + C3ISPFrameInfo *info = data->frameInfo_.find(buffer); + if (!info) + return; + + info->paramDequeued = true; + tryCompleteRequest(info); +} + +bool PipelineHandlerC3ISP::createCamera(MediaLink *ispLink) +{ + MediaEntity *adap = ispLink->source()->entity(); + const MediaPad *adapSink = adap->getPadByIndex(0); + MediaEntity *csi = adapSink->links()[0]->source()->entity(); + const MediaPad *csiSink = csi->getPadByIndex(0); + MediaEntity *sensor = csiSink->links()[0]->source()->entity(); + + std::unique_ptr data = + std::make_unique(this, sensor); + + if (data->init()) + return false; + + /* Generic values for sensor */ + std::unordered_map params = { + { V4L2_CID_ANALOGUE_GAIN, { 1, false } }, + { V4L2_CID_EXPOSURE, { 2, false } }, + }; + + data->delayedCtrls_ = std::make_unique(data->sensor_->device(), params); + isp_->frameStart.connect(data->delayedCtrls_.get(), &DelayedControls::applyControls); + + int ret = data->loadIPA(media_->hwRevision()); + if (ret) + return false; + + std::set streams{ &data->viewStream, &data->stillStream, &data->videoStream }; + + std::shared_ptr camera = Camera::create(std::move(data), sensor->name(), streams); + + registerCamera(std::move(camera)); + + return true; +} + +bool PipelineHandlerC3ISP::match(DeviceEnumerator *enumerator) +{ + const MediaPad *ispSink; + + DeviceMatch dm("c3-isp"); + dm.add("c3-mipi-csi2"); + dm.add("c3-mipi-adapter"); + dm.add("c3-isp-core"); + + media_ = acquireMediaDevice(enumerator, dm); + if (!media_) + return false; + + isp_ = V4L2Subdevice::fromEntityName(media_, "c3-isp-core"); + if (isp_->open() < 0) + return false; + + stat_ = V4L2VideoDevice::fromEntityName(media_, "c3-isp-stats"); + if (stat_->open() < 0) + return false; + + stat_->bufferReady.connect(this, &PipelineHandlerC3ISP::statReady); + + param_ = V4L2VideoDevice::fromEntityName(media_, "c3-isp-params"); + if (param_->open() < 0) + return false; + + param_->bufferReady.connect(this, &PipelineHandlerC3ISP::paramReady); + + C3ISPPipe *viewPipe = &pipes_[C3ISPVIEW]; + viewPipe->resizer = V4L2Subdevice::fromEntityName(media_, "c3-isp-resizer0"); + if (viewPipe->resizer->open() < 0) + return false; + + viewPipe->cap = V4L2VideoDevice::fromEntityName(media_, "c3-isp-cap0"); + if (viewPipe->cap->open() < 0) + return false; + + viewPipe->cap->bufferReady.connect(this, &PipelineHandlerC3ISP::bufferReady); + + C3ISPPipe *stillPipe = &pipes_[C3ISPSTILL]; + stillPipe->resizer = V4L2Subdevice::fromEntityName(media_, "c3-isp-resizer1"); + if (stillPipe->resizer->open() < 0) + return false; + + stillPipe->cap = V4L2VideoDevice::fromEntityName(media_, "c3-isp-cap1"); + if (stillPipe->cap->open() < 0) + return false; + + stillPipe->cap->bufferReady.connect(this, &PipelineHandlerC3ISP::bufferReady); + + C3ISPPipe *videoPipe = &pipes_[C3ISPVIDEO]; + videoPipe->resizer = V4L2Subdevice::fromEntityName(media_, "c3-isp-resizer2"); + if (videoPipe->resizer->open() < 0) + return false; + + videoPipe->cap = V4L2VideoDevice::fromEntityName(media_, "c3-isp-cap2"); + if (videoPipe->cap->open() < 0) + return false; + + videoPipe->cap->bufferReady.connect(this, &PipelineHandlerC3ISP::bufferReady); + + ispSink = isp_->entity()->getPadByIndex(0); + if (!ispSink || ispSink->links().empty()) + return false; + + if (!createCamera(ispSink->links()[0])) { + LOG(C3ISP, Error) << "Failed to create camera"; + return false; + } + + return true; +} + +REGISTER_PIPELINE_HANDLER(PipelineHandlerC3ISP, "c3isp") + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/c3-isp/meson.build b/src/libcamera/pipeline/c3-isp/meson.build new file mode 100644 index 00000000..5f8b23f1 --- /dev/null +++ b/src/libcamera/pipeline/c3-isp/meson.build @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: CC0-1.0 + +libcamera_internal_sources += files([ + 'c3-isp.cpp' +]) From patchwork Thu Dec 12 02:12:46 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Keke Li X-Patchwork-Id: 22293 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 5734CC3260 for ; Thu, 12 Dec 2024 02:13:02 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 2E23467EC5; Thu, 12 Dec 2024 03:13:02 +0100 (CET) Received: from mail-sh.amlogic.com (unknown [58.32.228.46]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1C7B867EBA for ; Thu, 12 Dec 2024 03:12:54 +0100 (CET) Received: from droid10.amlogic.com (10.18.11.213) by mail-sh.amlogic.com (10.18.11.5) with Microsoft SMTP Server id 15.1.2507.39; Thu, 12 Dec 2024 10:12:52 +0800 From: Keke Li To: CC: Keke Li Subject: [PATCH 2/2] libcamera: ipa: Add C3 ISP IPA Date: Thu, 12 Dec 2024 10:12:46 +0800 Message-ID: <20241212021246.115736-3-keke.li@amlogic.com> X-Mailer: git-send-email 2.29.0 In-Reply-To: <20241212021246.115736-1-keke.li@amlogic.com> References: <20241212021246.115736-1-keke.li@amlogic.com> MIME-Version: 1.0 X-Originating-IP: [10.18.11.213] X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The C3 ISP IPA is used to process 3A statistics and generate parameters for ISP hardware. Signed-off-by: Keke Li --- include/linux/c3-isp-config.h | 555 +++++++++++++++++++++++ meson_options.txt | 2 +- src/ipa/c3-isp/algorithms/agc.cpp | 260 +++++++++++ src/ipa/c3-isp/algorithms/agc.h | 50 ++ src/ipa/c3-isp/algorithms/algorithm.h | 31 ++ src/ipa/c3-isp/algorithms/awb.cpp | 257 +++++++++++ src/ipa/c3-isp/algorithms/awb.h | 42 ++ src/ipa/c3-isp/algorithms/blc.cpp | 102 +++++ src/ipa/c3-isp/algorithms/blc.h | 40 ++ src/ipa/c3-isp/algorithms/ccm.cpp | 86 ++++ src/ipa/c3-isp/algorithms/ccm.h | 40 ++ src/ipa/c3-isp/algorithms/csc.cpp | 64 +++ src/ipa/c3-isp/algorithms/csc.h | 32 ++ src/ipa/c3-isp/algorithms/meson.build | 10 + src/ipa/c3-isp/algorithms/post_gamma.cpp | 64 +++ src/ipa/c3-isp/algorithms/post_gamma.h | 33 ++ src/ipa/c3-isp/c3-isp.cpp | 386 ++++++++++++++++ src/ipa/c3-isp/data/imx290.yaml | 30 ++ src/ipa/c3-isp/data/meson.build | 9 + src/ipa/c3-isp/ipa_context.cpp | 251 ++++++++++ src/ipa/c3-isp/ipa_context.h | 110 +++++ src/ipa/c3-isp/meson.build | 32 ++ src/ipa/c3-isp/module.h | 28 ++ src/ipa/c3-isp/params.cpp | 127 ++++++ src/ipa/c3-isp/params.h | 133 ++++++ 25 files changed, 2773 insertions(+), 1 deletion(-) create mode 100644 include/linux/c3-isp-config.h create mode 100644 src/ipa/c3-isp/algorithms/agc.cpp create mode 100644 src/ipa/c3-isp/algorithms/agc.h create mode 100644 src/ipa/c3-isp/algorithms/algorithm.h create mode 100755 src/ipa/c3-isp/algorithms/awb.cpp create mode 100755 src/ipa/c3-isp/algorithms/awb.h create mode 100644 src/ipa/c3-isp/algorithms/blc.cpp create mode 100644 src/ipa/c3-isp/algorithms/blc.h create mode 100644 src/ipa/c3-isp/algorithms/ccm.cpp create mode 100644 src/ipa/c3-isp/algorithms/ccm.h create mode 100644 src/ipa/c3-isp/algorithms/csc.cpp create mode 100644 src/ipa/c3-isp/algorithms/csc.h create mode 100644 src/ipa/c3-isp/algorithms/meson.build create mode 100644 src/ipa/c3-isp/algorithms/post_gamma.cpp create mode 100644 src/ipa/c3-isp/algorithms/post_gamma.h create mode 100644 src/ipa/c3-isp/c3-isp.cpp create mode 100644 src/ipa/c3-isp/data/imx290.yaml create mode 100644 src/ipa/c3-isp/data/meson.build create mode 100644 src/ipa/c3-isp/ipa_context.cpp create mode 100644 src/ipa/c3-isp/ipa_context.h create mode 100644 src/ipa/c3-isp/meson.build create mode 100644 src/ipa/c3-isp/module.h create mode 100644 src/ipa/c3-isp/params.cpp create mode 100644 src/ipa/c3-isp/params.h diff --git a/include/linux/c3-isp-config.h b/include/linux/c3-isp-config.h new file mode 100644 index 00000000..007ef258 --- /dev/null +++ b/include/linux/c3-isp-config.h @@ -0,0 +1,555 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */ +/* + * Copyright (C) 2024 Amlogic, Inc. All rights reserved + */ + +#ifndef _UAPI_C3_ISP_CONFIG_H_ +#define _UAPI_C3_ISP_CONFIG_H_ + +#include + +/* + * Frames are split into zones of almost equal width and height - a zone is a + * rectangular tile of a frame. The metering blocks within the ISP collect + * aggregated statistics per zone. + */ +#define C3_ISP_AE_MAX_ZONES (17 * 15) +#define C3_ISP_AF_MAX_ZONES (17 * 15) +#define C3_ISP_AWB_MAX_ZONES (32 * 24) + +/* The maximum number of point on the diagonal of the frame for statistics */ +#define C3_ISP_AE_MAX_PT_NUM 18 +#define C3_ISP_AF_MAX_PT_NUM 18 +#define C3_ISP_AWB_MAX_PT_NUM 33 + +/** + * struct c3_isp_awb_zone_stats - AWB statistics of a zone + * + * AWB zone stats is aligned with 8 bytes + * + * @rg: the ratio of R / G in a zone + * @bg: the ratio of B / G in a zone + * @pixel_sum: the total number of pixels used in a zone + */ +struct c3_isp_awb_zone_stats { + __u16 rg; + __u16 bg; + __u32 pixel_sum; +}; + +/** + * struct c3_isp_awb_stats - Auto white balance statistics information. + * + * AWB statistical information of all zones. + * + * @stats: array of auto white balance statistics + */ +struct c3_isp_awb_stats { + struct c3_isp_awb_zone_stats stats[C3_ISP_AWB_MAX_ZONES]; +} __attribute__((aligned(16))); + +/** + * struct c3_isp_ae_zone_stats - AE statistics of a zone + * + * AE zone stats is aligned with 8 bytes. + * This is a 5-bin histogram and the total sum is + * normalized to 0xffff. + * So hist2 = 0xffff - (hist0 + hist1 + hist3 + hist4) + * + * @hist0: the global normalized pixel count for bin 0 + * @hist1: the global normalized pixel count for bin 1 + * @hist3: the global normalized pixel count for bin 3 + * @hist4: the global normalized pixel count for bin 4 + */ +struct c3_isp_ae_zone_stats { + __u16 hist0; + __u16 hist1; + __u16 hist3; + __u16 hist4; +}; + +/** + * struct c3_isp_ae_stats - Exposure statistics information + * + * AE statistical information consists of + * all blocks information and a 1024-bin histogram. + * + * @stats: array of auto exposure block statistics + * @reserved: undefined buffer space + * @hist: a 1024-bin histogram for the entire image + */ +struct c3_isp_ae_stats { + struct c3_isp_ae_zone_stats stats[C3_ISP_AE_MAX_ZONES]; + __u32 reserved[2]; + __u32 hist[1024]; +} __attribute__((aligned(16))); + +/** + * struct c3_isp_af_zone_stats - AF statistics of a zone + * + * AF zone stats is aligned with 8 bytes. + * The zonal accumulated contrast metrics are stored + * in floating point format with 16 bits mantissa and + * 5 or 6 bits exponent. + * Apart from contrast metrics we accumulate squared image and + * quartic image data over the zone. + * + * @i2_mat: the mantissa of zonal squared image pixel sum + * @i4_mat: the mantissa of zonal quartic image pixel sum + * @e4_mat: the mantissa of zonal multi-directional quartic edge sum + * @e4_exp: the exponent of zonal multi-directional quartic edge sum + * @i2_exp: the exponent of zonal squared image pixel sum + * @i4_exp: the exponent of zonal quartic image pixel sum + */ +struct c3_isp_af_zone_stats { + __u16 i2_mat; + __u16 i4_mat; + __u16 e4_mat; + __u16 e4_exp : 5; + __u16 i2_exp : 5; + __u16 i4_exp : 6; +}; + +/** + * struct c3_isp_af_stats - Auto Focus statistics information + * + * AF statistical information of each zone + * + * @stats: array of auto focus block statistics + * @reserved: undefined buffer space + */ +struct c3_isp_af_stats { + struct c3_isp_af_zone_stats stats[C3_ISP_AF_MAX_ZONES]; + __u32 reserved[2]; +} __attribute__((aligned(16))); + +/** + * struct c3_isp_stats_buffer - V4L2_META_FMT_C3ISP_STATS + * + * Contains ISP statistics + * + * @awb: auto white balance stats + * @ae: auto exposure stats + * @af: auto focus stats + */ +struct c3_isp_stats_buffer { + struct c3_isp_awb_stats awb; + struct c3_isp_ae_stats ae; + struct c3_isp_af_stats af; +}; + +/** + * enum c3_isp_params_buffer_version - C3 ISP parameters block versioning + * + * @C3_ISP_PARAMS_BUFFER_V0: First version of C3 ISP parameters block + */ +enum c3_isp_params_buffer_version { + C3_ISP_PARAMS_BUFFER_V0, +}; + +/** + * enum c3_isp_params_block_type - Enumeration of C3 ISP parameter blocks + * + * Each block configures a specific processing block of the C3 ISP. + * The block type allows the driver to correctly interpret + * the parameters block data. + * + * @C3_ISP_PARAMS_BLOCK_AWB_GAINS: White balance gains + * @C3_ISP_PARAMS_BLOCK_AWB_CONFIG: AE statistics parameters + * @C3_ISP_PARAMS_BLOCK_AE_CONFIG: AE statistics parameters + * @C3_ISP_PARAMS_BLOCK_AF_CONFIG: AF statistics parameters + * @C3_ISP_PARAMS_BLOCK_PST_GAMMA: post gamma parameters + * @C3_ISP_PARAMS_BLOCK_CCM: Color correction matrix parameters + * @C3_ISP_PARAMS_BLOCK_CSC: Color space conversion parameters + * @C3_ISP_PARAMS_BLOCK_BLC: Black level correction parameters + * @C3_ISP_PARAMS_BLOCK_SENTINEL: First non-valid block index + */ +enum c3_isp_params_block_type { + C3_ISP_PARAMS_BLOCK_AWB_GAINS, + C3_ISP_PARAMS_BLOCK_AWB_CONFIG, + C3_ISP_PARAMS_BLOCK_AE_CONFIG, + C3_ISP_PARAMS_BLOCK_AF_CONFIG, + C3_ISP_PARAMS_BLOCK_PST_GAMMA, + C3_ISP_PARAMS_BLOCK_CCM, + C3_ISP_PARAMS_BLOCK_CSC, + C3_ISP_PARAMS_BLOCK_BLC, + C3_ISP_PARAMS_BLOCK_SENTINEL +}; + +#define C3_ISP_PARAMS_BLOCK_FL_NONE 0 +#define C3_ISP_PARAMS_BLOCK_FL_DISABLED BIT(0) + +/** + * struct c3_isp_params_block_header - C3 ISP parameter block header + * + * This structure represents the common part of all the ISP configuration + * blocks. Each parameters block shall embed an instance of this structure type + * as its first member, followed by the block-specific configuration data. The + * driver inspects this common header to discern the block type and its size and + * properly handle the block content by casting it to the correct block-specific + * type. + * + * The @type field is one of the values enumerated by + * :c:type:`c3_isp_params_block_type` and specifies how the data should be + * interpreted by the driver. The @size field specifies the size of the + * parameters block and is used by the driver for validation purposes. The + * @flags field is a bitmask of per-block flags C3_ISP_PARAMS_FL*. + * + * When userspace wants to disable an ISP block the + * C3_ISP_PARAMS_BLOCK_FL_DISABLED bit should be set in the @flags field. In + * this case userspace may optionally omit the remainder of the configuration + * block, which will be ignored by the driver. + * + * When a new configuration of an ISP block needs to be applied userspace + * shall fully populate the ISP block and omit setting the + * C3_ISP_PARAMS_BLOCK_FL_DISABLED bit in the @flags field. + * + * Userspace is responsible for correctly populating the parameters block header + * fields (@type, @flags and @size) and the block-specific parameters. + * + * For example: + * + * .. code-block:: c + * + * void populate_pst_gamma(struct c3_isp_params_block_header *block) { + * struct c3_isp_params_pst_gamma *gamma = + * (struct c3_isp_params_pst_gamma *)block; + * + * gamma->header.type = C3_ISP_PARAMS_BLOCK_PST_GAMMA; + * gamma->header.flags = C3_ISP_PARAMS_BLOCK_FL_NONE; + * gamma->header.size = sizeof(*gamma); + * + * for (unsigned int i = 0; i < 129; i++) + * gamma->pst_gamma_lut[i] = i; + * } + * + * @type: The parameters block type from :c:type:`c3_isp_params_block_type` + * @flags: A bitmask of block flags + * @size: Size (in bytes) of the parameters block, including this header + */ +struct c3_isp_params_block_header { + __u16 type; + __u16 flags; + __u32 size; +}; + +/** + * struct c3_isp_params_awb_gains - Gains for auto-white balance + * + * This struct allows users to configure the gains for white balance. There + * are four gain settings corresponding to each colour channel in the bayer domain. + * All of the gains are stored in Q4.8 format. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_AWB_GAINS + * from :c:type:`c3_isp_params_block_type` + * + * @header: The C3 ISP parameters block header + * @gr_gain: Multiplier for Gr channel (Q4.8 format) + * @r_gain: Multiplier for R channel (Q4.8 format) + * @b_gain: Multiplier for B channel (Q4.8 format) + * @gb_gain: Multiplier for Gb channel (Q4.8 format) + */ +struct c3_isp_params_awb_gains { + struct c3_isp_params_block_header header; + __u16 gr_gain; + __u16 r_gain; + __u16 b_gain; + __u16 gb_gain; +}; + +/* + * enum c3_isp_params_awb_tap_points - Tap points for the AWB statistics + * @C3_ISP_AWB_STATS_TAP_FE: immediately after the optical frontend block + * @C3_ISP_AWB_STATS_TAP_GE: immediately after the green equal block + * @C3_ISP_AWB_STATS_TAP_BEFORE_WB: immediately before the white balance block + * @C3_ISP_AWB_STATS_TAP_AFTER_WB: immediately after the white balance block + */ +enum c3_isp_params_awb_tap_point { + C3_ISP_AWB_STATS_TAP_OFE = 0, + C3_ISP_AWB_STATS_TAP_GE, + C3_ISP_AWB_STATS_TAP_BEFORE_WB, + C3_ISP_AWB_STATS_TAP_AFTER_WB, +}; + +/** + * struct c3_isp_params_awb_config - Stats settings for auto-white balance + * + * This struct allows the configuration of the statistics generated for auto + * white balance. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_AWB_CONFIG + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @tap_point: the tap point from enum c3_isp_params_awb_tap_point + * @satur_vald: AWB statistic over saturation control + * value: 0: disable, 1: enable + * @horiz_zones_num: active zones number of hotizontal [0..32] + * @vert_zones_num: active zones number of vertical [0..24] + * @rg_min: minimum R/G ratio (Q4.8 format) + * @rg_max: maximum R/G ratio (Q4.8 format) + * @bg_min: minimum B/G ratio (Q4.8 format) + * @bg_max: maximum B/G ratio (Q4.8 format) + * @rg_low: R/G ratio trim low (Q4.8 format) + * @rg_high: R/G ratio trim hight (Q4.8 format) + * @bg_low: B/G ratio trim low (Q4.8 format) + * @bg_high: B/G ratio trim high (Q4.8 format) + * @zone_weight: array of weights for AWB statistics zones [0..15] + * @horiz_cood: the horizontal coordinate of point on the diagonal [0..2888] + * @vert_cood: the vertical coordinate of point on the diagonal [0..2240] + */ +struct c3_isp_params_awb_config { + struct c3_isp_params_block_header header; + __u8 tap_point; + __u8 satur_vald; + __u8 horiz_zones_num; + __u8 vert_zones_num; + __u16 rg_min; + __u16 rg_max; + __u16 bg_min; + __u16 bg_max; + __u16 rg_low; + __u16 rg_high; + __u16 bg_low; + __u16 bg_high; + __u8 zone_weight[C3_ISP_AWB_MAX_ZONES]; + __u16 horiz_cood[C3_ISP_AWB_MAX_PT_NUM]; + __u16 vert_cood[C3_ISP_AWB_MAX_PT_NUM]; +}; + +/* + * enum c3_isp_params_ae_tap_points - Tap points for the AE statistics + * @C3_ISP_AE_STATS_TAP_GE: immediately after the green equal block + * @C3_ISP_AE_STATS_TAP_MLS: immediately after the mesh lens shading block + */ +enum c3_isp_params_ae_tap_point { + C3_ISP_AE_STATS_TAP_GE = 0, + C3_ISP_AE_STATS_TAP_MLS, +}; + +/** + * struct c3_isp_params_ae_config - Stats settings for auto-exposure + * + * This struct allows the configuration of the statistics generated for + * auto exposure. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_AE_CONFIG + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @horiz_zones_num: active zones number of hotizontal [0..17] + * @vert_zones_num: active zones number of vertical [0..15] + * @tap_point: the tap point from enum c3_isp_params_ae_tap_point + * @zone_weight: array of weights for AE statistics zones [0..15] + * @horiz_cood: the horizontal coordinate of point on the diagonal [0..2888] + * @vert_cood: the vertical coordinate of point on the diagonal [0..2240] + */ +struct c3_isp_params_ae_config { + struct c3_isp_params_block_header header; + __u8 tap_point; + __u8 horiz_zones_num; + __u8 vert_zones_num; + __u8 zone_weight[C3_ISP_AE_MAX_ZONES]; + __u16 horiz_cood[C3_ISP_AE_MAX_PT_NUM]; + __u16 vert_cood[C3_ISP_AE_MAX_PT_NUM]; +}; + +/* + * enum c3_isp_params_af_tap_points - Tap points for the AF statistics + * @C3_ISP_AF_STATS_TAP_SNR: immediately after the spatial noise reduce block + * @C3_ISP_AF_STATS_TAP_DMS: immediately after the demosaic block + */ +enum c3_isp_params_af_tap_point { + C3_ISP_AF_STATS_TAP_SNR = 0, + C3_ISP_AF_STATS_TAP_DMS, +}; + +/** + * struct c3_isp_params_af_config - Stats settings for auto-focus + * + * This struct allows the configuration of the statistics generated for + * auto focus. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_AF_CONFIG + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @tap_point: the tap point from enum c3_isp_params_af_tap_point + * @horiz_zones_num: active zones number of hotizontal [0..17] + * @vert_zones_num: active zones number of vertical [0..15] + * @horiz_cood: the horizontal coordinate of point on the diagonal [0..2888] + * @vert_cood: the vertical coordinate of point on the diagonal [0..2240] + */ +struct c3_isp_params_af_config { + struct c3_isp_params_block_header header; + __u8 tap_point; + __u8 horiz_zones_num; + __u8 vert_zones_num; + __u16 horiz_cood[C3_ISP_AF_MAX_PT_NUM]; + __u16 vert_cood[C3_ISP_AF_MAX_PT_NUM]; +}; + +/** + * struct c3_isp_params_pst_gamma - Post gamma configuration + * + * This struct allows the configuration of the look up table for + * post gamma. The gamma curve consists of 129 points, so need to + * set lut[129]. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_PST_GAMMA + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @lut: look up table for P-Stitch gamma [0..1023] + */ +struct c3_isp_params_pst_gamma { + struct c3_isp_params_block_header header; + __u16 lut[129]; +}; + +/** + * struct c3_isp_params_ccm - ISP CCM configuration + * + * This struct allows the configuration of the matrix for + * color correction. The matrix consists of 3 x 3 points, + * so need to set matrix[3][3]. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_CCM + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @matrix: a 3 x 3 matrix used for color correction, + * the value of matrix[x][y] is orig_value x 256. [-4096..4095] + */ +struct c3_isp_params_ccm { + struct c3_isp_params_block_header header; + __s16 matrix[3][3]; +}; + +/** + * struct c3_isp_params_csc - ISP Color Space Conversion configuration + * + * This struct allows the configuration of the matrix for + * color space conversion. The matrix consists of 3 x 3 points, + * so need to set matrix[3][3]. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_CSC + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @matrix: a 3x3 matrix used for the color space conversion, + * the value of matrix[x][y] is orig_value x 256. [-4096..4095] + */ +struct c3_isp_params_csc { + struct c3_isp_params_block_header header; + __s16 matrix[3][3]; +}; + +/** + * struct c3_isp_params_blc - ISP Black Level Correction configuration + * + * This struct allows the configuration of the block level offset + * for each color channel. + * + * header.type should be set to C3_ISP_PARAMS_BLOCK_BLC + * from :c:type:`c3_isp_params_block_type` + * + * @header: the C3 ISP parameters block header + * @gr_ofst: Gr blc offset (Q4.8 format) + * @r_ofst: R blc offset (Q4.8 format) + * @b_ofst: B blc offset (Q4.8 format) + * @gb_ofst: Gb blc offset(Q4.8 format) + */ +struct c3_isp_params_blc { + struct c3_isp_params_block_header header; + __u16 gr_ofst; + __u16 r_ofst; + __u16 b_ofst; + __u16 gb_ofst; +}; + +/** + * define C3_ISP_PARAMS_MAX_SIZE - Maximum size of all C3 ISP Parameters + * + * Though the parameters for the C3 ISP are passed as optional blocks, the + * driver still needs to know the absolute maximum size so that it can allocate + * a buffer sized appropriately to accommodate userspace attempting to set all + * possible parameters in a single frame. + */ +#define C3_ISP_PARAMS_MAX_SIZE \ + (sizeof(struct c3_isp_params_awb_gains) + \ + sizeof(struct c3_isp_params_awb_config) + \ + sizeof(struct c3_isp_params_ae_config) + \ + sizeof(struct c3_isp_params_af_config) + \ + sizeof(struct c3_isp_params_pst_gamma) + \ + sizeof(struct c3_isp_params_ccm) + \ + sizeof(struct c3_isp_params_csc) + \ + sizeof(struct c3_isp_params_blc)) + +/** + * struct c3_isp_params_cfg - C3 ISP configuration parameters + * + * This struct contains the configuration parameters of the C3 ISP + * algorithms, serialized by userspace into an opaque data buffer. Each + * configuration parameter block is represented by a block-specific structure + * which contains a :c:type:`c3_isp_param_block_header` entry as first + * member. Userspace populates the @data buffer with configuration parameters + * for the blocks that it intends to configure. As a consequence, the data + * buffer effective size changes according to the number of ISP blocks that + * userspace intends to configure. + * + * The parameters buffer is versioned by the @version field to allow modifying + * and extending its definition. Userspace should populate the @version field to + * inform the driver about the version it intends to use. The driver will parse + * and handle the @data buffer according to the data layout specific to the + * indicated revision and return an error if the desired revision is not + * supported. + * + * For each ISP block that userspace wants to configure, a block-specific + * structure is appended to the @data buffer, one after the other without gaps + * in between nor overlaps. Userspace shall populate the @total_size field with + * the effective size, in bytes, of the @data buffer. + * + * The expected memory layout of the parameters buffer is:: + * + * +-------------------- struct c3_isp_params_cfg ---- ------------------+ + * | version = C3_ISP_PARAM_BUFFER_V0; | + * | data_size = sizeof(struct c3_isp_params_awb_gains) + | + * | sizeof(struct c3_isp_params_awb_config); | + * | +------------------------- data ---------------------------------+ | + * | | +------------ struct c3_isp_params_awb_gains) ------------------+ | + * | | | +--------- struct c3_isp_params_block_header header -----+ | | | + * | | | | type = C3_ISP_PARAMS_BLOCK_AWB_GAINS; | | | | + * | | | | flags = C3_ISP_PARAMS_BLOCK_FL_NONE; | | | | + * | | | | size = sizeof(struct c3_isp_params_awb_gains); | | | | + * | | | +---------------------------------------------------------+ | | | + * | | | gr_gain = ...; | | | + * | | | r_gain = ...; | | | + * | | | b_gain = ...; | | | + * | | | gb_gain = ...; | | | + * | | +------------------ struct c3_isp_params_awb_config ----------+ | | + * | | | +---------- struct c3_isp_param_block_header header ------+ | | | + * | | | | type = C3_ISP_PARAMS_BLOCK_AWB_CONFIG; | | | | + * | | | | flags = C3_ISP_PARAMS_BLOCK_FL_NONE; | | | | + * | | | | size = sizeof(struct c3_isp_params_awb_config) | | | | + * | | | +---------------------------------------------------------+ | | | + * | | | tap_point = ...; | | | + * | | | satur_vald = ...; | | | + * | | | horiz_zones_num = ...; | | | + * | | | vert_zones_num = ...; | | | + * | | +-------------------------------------------------------------+ | | + * | +-----------------------------------------------------------------+ | + * +---------------------------------------------------------------------+ + * + * @version: The C3 ISP parameters buffer version + * @data_size: The C3 ISP configuration data effective size, + * excluding this header + * @data: The C3 ISP configuration blocks data + */ +struct c3_isp_params_cfg { + __u8 version; + __u32 data_size; + __u8 data[C3_ISP_PARAMS_MAX_SIZE]; +}; + +#endif diff --git a/meson_options.txt b/meson_options.txt index bd782830..c4306f00 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -32,7 +32,7 @@ option('gstreamer', option('ipas', type : 'array', - choices : ['ipu3', 'rkisp1', 'rpi/vc4', 'simple', 'vimc'], + choices : ['c3-isp', 'ipu3', 'rkisp1', 'rpi/vc4', 'simple', 'vimc'], description : 'Select which IPA modules to build') option('lc-compliance', diff --git a/src/ipa/c3-isp/algorithms/agc.cpp b/src/ipa/c3-isp/algorithms/agc.cpp new file mode 100644 index 00000000..35674be3 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/agc.cpp @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP AGC/AEC mean-based control algorithm + */ + +#include "agc.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include "libcamera/internal/yaml_parser.h" + +#include "libipa/histogram.h" + +/** + * \file agc.h + */ + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +namespace ipa::c3isp::algorithms { + +/** + * \class Agc + * \brief A mean-based auto-exposure algorithm + */ + +LOG_DEFINE_CATEGORY(C3ISPAgc) + +Agc::Agc() + : horizonalZonesNum_(17), verticalZonesNum_(15) +{ +} + +/** + * \brief Initialise the AGC algorithm from tuning files + * \param[in] context The shared IPA context + * \param[in] tuningData The YamlObject containing Agc tuning data + * + * This function calls the base class' tuningData parsers to discover which + * control values are supported. + * + * \return 0 on success or errors from the base class + */ +int Agc::init(IPAContext &context, + const YamlObject &tuningData) +{ + int ret; + + ret = parseTuningData(tuningData); + if (ret) + return ret; + + context.ctrlMap.merge(controls()); + + return 0; +} + +/** + * \brief Configure the AGC given a configInfo + * \param[in] context The shared IPA context + * \param[in] configInfo The IPA configuration data + * + * \return 0 + */ +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + const IPASessionConfiguration &configuration = context.configuration; + IPAActiveState &activeState = context.activeState; + + /* Configure the default gain and exposure */ + activeState.agc.gain = configuration.sensor.minAnalogueGain; + activeState.agc.exposure = 10ms / configuration.sensor.lineDuration; + + activeState.agc.constraintMode = constraintModes().begin()->first; + activeState.agc.exposureMode = exposureModeHelpers().begin()->first; + + setLimits(configuration.sensor.minShutterSpeed, + configuration.sensor.maxShutterSpeed, + configuration.sensor.minAnalogueGain, + configuration.sensor.maxAnalogueGain); + + resetFrameCount(); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Agc::prepare(IPAContext &context, const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + C3ISPParams *params) +{ + if (frame) + return; + + auto AeCfg = params->block(); + AeCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + AeCfg->tap_point = C3_ISP_AE_STATS_TAP_MLS; + AeCfg->horiz_zones_num = horizonalZonesNum_; + AeCfg->vert_zones_num = verticalZonesNum_; + + for (unsigned int i = 0; i < AeCfg->horiz_zones_num * AeCfg->vert_zones_num; i++) + AeCfg->zone_weight[i] = 1; + + Size sensorSize = context.configuration.sensor.size; + uint8_t maxPointNum = std::max(AeCfg->horiz_zones_num, AeCfg->vert_zones_num) + 1; + + for (unsigned int i = 0; i < maxPointNum; i++) { + uint16_t hidx = i * sensorSize.width / AeCfg->horiz_zones_num; + + /* Aligned with 2 */ + hidx = hidx / 2 * 2; + AeCfg->horiz_cood[i] = std::min(hidx, (uint16_t)sensorSize.width); + + uint16_t vidx = i * sensorSize.height / AeCfg->vert_zones_num; + + /* Aligned with 2 */ + vidx = vidx / 2 * 2; + AeCfg->vert_cood[i] = std::min(vidx, (uint16_t)sensorSize.height); + } +} + +Histogram Agc::parseStatistics(const c3_isp_stats_buffer *stats) +{ + const struct c3_isp_ae_stats *info = &stats->ae; + uint16_t zonesNum = horizonalZonesNum_ * verticalZonesNum_; + std::vector means(zonesNum, 0); + + /* + * Each zone has a 5-bin histogram and the + * total sum is normalized to 65535. + * For the convenience of calculation, + * it can be assumed that: + * hist0 represents the number of brightness 0, + * hist1 represents the number of brightness 64, + * hist2 represents the number of brightness 128, + * hist3 represents the number of brightness 192, + * hist4 represents the number of brightness 255. + * + * Finally, the average brightness of a zone can be calculated. + */ + for (unsigned int i = 0; i < zonesNum; i++) { + uint16_t hist2 = 65535 - info->stats[i].hist0 - info->stats[i].hist1 - info->stats[i].hist3 - info->stats[i].hist4; + + uint32_t lumaSum = info->stats[i].hist0 * 0 + info->stats[i].hist1 * 64 + hist2 * 128 + info->stats[i].hist3 * 192 + info->stats[i].hist4 * 255; + + means[i] = lumaSum / 65535; + } + + lumaMeans_ = means; + + /* This is a 1024-bin histogram */ + uint32_t *hist = const_cast(info->hist); + + return Histogram(Span(hist, std::size(info->hist))); +} + +/** + * \brief Estimate the relative luminance of the frame with a given gain + * \param[in] gain The gain to apply in estimating luminance + * + * The estimation is based on the average value of the zone. Each + * average value is multiplied by the gain, and then saturated to + * to approximate the sensor behaviour at high brightness values. + * The approximation is quite rough, as it doesn't take into account + * non-linearities when approaching saturation. + * + * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds + * to a theoretical perfect reflector of 100% reference white. + * + * More detailed information can be found in: + * https://en.wikipedia.org/wiki/Relative_luminance + * + * \return The relative luminance of the frame + */ +double Agc::estimateLuminance(double gain) const +{ + double sum = 0.0; + + for (unsigned int i = 0; i < lumaMeans_.size(); i++) + sum += std::min(lumaMeans_[i] * gain, 255.0); + + return sum / lumaMeans_.size() / 255; +} +/** + * \brief Process C3 ISP statistics, and run AGC operations + * \param[in] context The shared IPA context + * \param[in] frame The current frame sequence number + * \param[in] frameContext The current frame context + * \param[in] stats The C3 ISP statistics and ISP results + * \param[out] metadata Metadata for the frame, to be filled by the algorithm + * + * Identify the current image brightness, and use that to estimate the optimal + * new exposure and gain for the scene. + */ +void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const c3_isp_stats_buffer *stats, + ControlList &metadata) +{ + Histogram hist = parseStatistics(stats); + + utils::Duration shutterTime; + double aGain, dGain; + + /* + * The Agc algorithm needs to know the effective exposure value that was + * applied to the sensor when the statistics were collected. + */ + utils::Duration exposureTime = context.configuration.sensor.lineDuration * frameContext.sensor.exposure; + double analogueGain = frameContext.sensor.gain; + utils::Duration effectiveExposureValue = exposureTime * analogueGain; + + std::tie(shutterTime, aGain, dGain) = + calculateNewEv(context.activeState.agc.constraintMode, + context.activeState.agc.exposureMode, hist, + effectiveExposureValue); + + LOG(C3ISPAgc, Debug) + << "Shutter time, analogue gain and digital gain are " + << shutterTime << ", " << aGain << " and " << dGain; + + IPAActiveState &activeState = context.activeState; + + activeState.agc.exposure = shutterTime / context.configuration.sensor.lineDuration; + activeState.agc.gain = aGain; + + metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + metadata.set(controls::ExposureTime, exposureTime.get()); + + uint32_t vTotal = context.configuration.sensor.size.height + context.configuration.sensor.defVBlank; + utils::Duration frameDuration = context.configuration.sensor.lineDuration * vTotal; + metadata.set(controls::FrameDuration, frameDuration.get()); + + lumaMeans_ = {}; +} + +REGISTER_IPA_ALGORITHM(Agc, "Agc") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/agc.h b/src/ipa/c3-isp/algorithms/agc.h new file mode 100644 index 00000000..d67092c2 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/agc.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP AGC/AEC mean-based control algorithm + */ + +#pragma once + +#include + +#include +#include + +#include + +#include "libipa/agc_mean_luminance.h" +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class Agc : public Algorithm, public AgcMeanLuminance +{ +public: + Agc(); + ~Agc() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const c3_isp_stats_buffer *stats, + ControlList &metadata) override; +private: + Histogram parseStatistics(const c3_isp_stats_buffer *stats); + double estimateLuminance(double gain) const override; + + std::vector lumaMeans_; + uint8_t horizonalZonesNum_; + uint8_t verticalZonesNum_; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/algorithm.h b/src/ipa/c3-isp/algorithms/algorithm.h new file mode 100644 index 00000000..eb40512c --- /dev/null +++ b/src/ipa/c3-isp/algorithms/algorithm.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP control algorithm interface + */ + +#pragma once + +#include + +#include "module.h" + +namespace libcamera { + +namespace ipa::c3isp { + +class Algorithm : public libcamera::ipa::Algorithm +{ +public: + Algorithm() + : disabled_(false) + { + } + + bool disabled_; +}; + +} /* namespace ipa::c3isp */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/awb.cpp b/src/ipa/c3-isp/algorithms/awb.cpp new file mode 100755 index 00000000..cbeccc3e --- /dev/null +++ b/src/ipa/c3-isp/algorithms/awb.cpp @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP AWB control algorithm + */ + +#include "awb.h" + +#include +#include + +#include + +#include + +#include + +/** + * \file awb.h + */ + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +/** + * \class Awb + * \brief A Grey world white balance correction algorithm + */ + +LOG_DEFINE_CATEGORY(C3ISPAwb) + +Awb::Awb() + : horizonalZonesNum_(32), verticalZonesNum_(24) +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int Awb::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + IPAActiveState &activeState = context.activeState; + + activeState.awb.gains.manual.red = 1.0; + activeState.awb.gains.manual.blue = 1.0; + activeState.awb.gains.manual.green = 1.0; + + activeState.awb.gains.automatic.red = 1.0; + activeState.awb.gains.automatic.blue = 1.0; + activeState.awb.gains.automatic.green = 1.0; + activeState.awb.autoEnabled = true; + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::queueRequest + */ +void Awb::queueRequest(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) +{ + auto &awb = context.activeState.awb; + + const auto &awbEnable = controls.get(controls::AwbEnable); + if (awbEnable && *awbEnable != awb.autoEnabled) { + awb.autoEnabled = *awbEnable; + + LOG(C3ISPAwb, Debug) + << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; + } + + const auto &colourGains = controls.get(controls::ColourGains); + if (colourGains && !awb.autoEnabled) { + awb.gains.manual.red = (*colourGains)[0]; + awb.gains.manual.blue = (*colourGains)[1]; + + LOG(C3ISPAwb, Debug) + << "Set colour gains to red: " << awb.gains.manual.red + << ", blue: " << awb.gains.manual.blue; + } + + frameContext.awb.autoEnabled = awb.autoEnabled; + + if (!awb.autoEnabled) { + frameContext.awb.gains.red = awb.gains.manual.red; + frameContext.awb.gains.green = 1.0; + frameContext.awb.gains.blue = awb.gains.manual.blue; + } +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Awb::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, C3ISPParams *params) +{ + /* + * This is the latest time we can read the active state. This is the + * most up-to-date automatic values we can read. + */ + if (frameContext.awb.autoEnabled) { + frameContext.awb.gains.red = context.activeState.awb.gains.automatic.red; + frameContext.awb.gains.green = context.activeState.awb.gains.automatic.green; + frameContext.awb.gains.blue = context.activeState.awb.gains.automatic.blue; + } + + auto AWBGains = params->block(); + AWBGains.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + AWBGains->gr_gain = std::clamp(256 * frameContext.awb.gains.green, 0, 0xfff); + AWBGains->r_gain = std::clamp(256 * frameContext.awb.gains.red, 0, 0xfff); + AWBGains->b_gain = std::clamp(256 * frameContext.awb.gains.blue, 0, 0xfff); + AWBGains->gb_gain = std::clamp(256 * frameContext.awb.gains.green, 0, 0xfff); + + if (frame) + return; + + auto AWBCfg = params->block(); + AWBCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + AWBCfg->tap_point = C3_ISP_AWB_STATS_TAP_BEFORE_WB; + AWBCfg->satur_vald = 1; + AWBCfg->horiz_zones_num = horizonalZonesNum_; + AWBCfg->vert_zones_num = verticalZonesNum_; + AWBCfg->rg_min = 75; + AWBCfg->rg_max = 256; + AWBCfg->bg_min = 44; + AWBCfg->bg_max = 222; + AWBCfg->rg_low = 93; + AWBCfg->rg_high = 244; + AWBCfg->bg_low = 61; + AWBCfg->bg_high = 205; + + for (unsigned int i = 0; i < AWBCfg->horiz_zones_num * AWBCfg->vert_zones_num; i++) + AWBCfg->zone_weight[i] = 1; + + Size sensorSize = context.configuration.sensor.size; + uint8_t maxPointNum = std::max(AWBCfg->horiz_zones_num, AWBCfg->vert_zones_num) + 1; + + for (unsigned int i = 0; i < maxPointNum; i++) { + uint16_t hidx = i * sensorSize.width / AWBCfg->horiz_zones_num; + + /* Aligned with 2 */ + hidx = hidx / 2 * 2; + AWBCfg->horiz_cood[i] = std::min(hidx, (uint16_t)sensorSize.width); + + uint16_t vidx = i * sensorSize.height / AWBCfg->vert_zones_num; + + /* Aligned with 2 */ + vidx = vidx / 2 * 2; + AWBCfg->vert_cood[i] = std::min(vidx, (uint16_t)sensorSize.height); + } +} + +uint32_t Awb::estimateCCT(double red, double green, double blue) +{ + /* Convert the RGB values to CIE tristimulus values (XYZ) */ + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); + + /* Calculate the normalized chromaticity values */ + double x = X / (X + Y + Z); + double y = Y / (X + Y + Z); + + /* Calculate CCT */ + double n = (x - 0.3320) / (0.1858 - y); + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Awb::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] const c3_isp_stats_buffer *stats, + ControlList &metadata) +{ + IPAActiveState &activeState = context.activeState; + const struct c3_isp_awb_stats *awb = &stats->awb; + uint16_t zoneCnt = horizonalZonesNum_ * verticalZonesNum_; + uint32_t rgSum = 0; + uint32_t bgSum = 0; + double rgMean; + double bgMean; + double greenMean; + double blueMean; + double redMean; + + for (unsigned int i = 0; i < zoneCnt; i++) { + rgSum += awb->stats[i].rg; + bgSum += awb->stats[i].bg; + } + + rgMean = rgSum / zoneCnt / 4096.0; + bgMean = bgSum / zoneCnt / 4096.0; + + /* + * To simplify the calculation, + * the green mean is hardcoded to 1.0 + */ + + greenMean = 1.0; + redMean = rgMean * greenMean; + blueMean = bgMean * greenMean; + + activeState.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); + + /* Metadata shall contain the up to date measurement */ + metadata.set(controls::ColourTemperature, activeState.awb.temperatureK); + + /* + * Estimate the red and blue gains to apply in a grey world. + * The green gain is hardcoded to 1.0. Avoid division by zero + * by clamping the divisor to mininum value of 0.0625. + */ + double redGain = greenMean / std::max(redMean, 0.0625); + double blueGain = greenMean / std::max(blueMean, 0.0625); + + /* + * Clamp the gain values to the hardware, which express gains as Q4.8 + * unsigned integer values. Set the minimum just above zero to avoid + * divisions by zero. + */ + redGain = std::clamp(redGain, 1.0 / 256, 4095.0 / 256); + blueGain = std::clamp(blueGain, 1.0 / 256, 4095.0 / 256); + + /* Filter the values to avoid oscillations. */ + double speed = 0.2; + redGain = speed * redGain + (1 - speed) * activeState.awb.gains.automatic.red; + blueGain = speed * blueGain + (1 - speed) * activeState.awb.gains.automatic.blue; + + activeState.awb.gains.automatic.red = redGain; + activeState.awb.gains.automatic.blue = blueGain; + activeState.awb.gains.automatic.green = 1.0; + + metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); + metadata.set(controls::ColourGains, { static_cast(frameContext.awb.gains.red), + static_cast(frameContext.awb.gains.blue) }); + + LOG(C3ISPAwb, Debug) << "Gains: R " << activeState.awb.gains.automatic.red + << ", G " << activeState.awb.gains.automatic.green + << ", B " << activeState.awb.gains.automatic.blue + << ", Ct " << activeState.awb.temperatureK << "K"; +} + +REGISTER_IPA_ALGORITHM(Awb, "Awb") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/awb.h b/src/ipa/c3-isp/algorithms/awb.h new file mode 100755 index 00000000..5f0cbfbb --- /dev/null +++ b/src/ipa/c3-isp/algorithms/awb.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP AWB control algorithm + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class Awb : public Algorithm +{ +public: + Awb(); + ~Awb() = default; + + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; + void queueRequest(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const c3_isp_stats_buffer *stats, + ControlList &metadata) override; + +private: + uint32_t estimateCCT(double red, double green, double blue); + uint8_t horizonalZonesNum_; + uint8_t verticalZonesNum_; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/blc.cpp b/src/ipa/c3-isp/algorithms/blc.cpp new file mode 100644 index 00000000..3c364596 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/blc.cpp @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Black Level Correction control + */ + +#include "blc.h" + +#include +#include + +#include "libcamera/internal/yaml_parser.h" + +/** + * \file blc.h + */ + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +/** + * \class Blc + * \brief C3 ISP Black Level Correction control + * + * The pixels output by the camera normally include a black level, because + * sensors do not always report a signal level of '0' for black. Pixels at or + * below this level should be considered black. To achieve that, the C3 ISP BLC + * algorithm subtracts a configurable offset from all pixels. + * + * The black level can be measured at runtime from an optical dark region of the + * camera sensor, or measured during the camera tuning process. The first option + * isn't currently supported. + */ + +LOG_DEFINE_CATEGORY(C3ISPBlc) + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Blc::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + std::optional levelRed = tuningData["BlcR"].get(); + std::optional levelGreenR = tuningData["BlcGr"].get(); + std::optional levelGreenB = tuningData["BlcGb"].get(); + std::optional levelBlue = tuningData["BlcB"].get(); + + blackLevelRed_ = levelRed.value_or(4096); + blackLevelGreenR_ = levelGreenR.value_or(4096); + blackLevelGreenB_ = levelGreenB.value_or(4096); + blackLevelBlue_ = levelBlue.value_or(4096); + + LOG(C3ISPBlc, Debug) + << "Black Levels: red " << blackLevelRed_ + << ", green (red) " << blackLevelGreenR_ + << ", green (blue) " << blackLevelGreenB_ + << ", blue " << blackLevelBlue_; + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Blc::prepare([[maybe_unused]] IPAContext &context, const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + C3ISPParams *params) +{ + if (frame) + return; + + auto blcCfg = params->block(); + blcCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + blcCfg->gr_ofst = blackLevelGreenR_; + blcCfg->r_ofst = blackLevelRed_; + blcCfg->b_ofst = blackLevelBlue_; + blcCfg->gb_ofst = blackLevelGreenB_; +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Blc::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const c3_isp_stats_buffer *stats, + ControlList &metadata) +{ + metadata.set(controls::SensorBlackLevels, + { static_cast(blackLevelRed_), + static_cast(blackLevelGreenR_), + static_cast(blackLevelGreenB_), + static_cast(blackLevelBlue_) }); +} + +REGISTER_IPA_ALGORITHM(Blc, "Blc") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/blc.h b/src/ipa/c3-isp/algorithms/blc.h new file mode 100644 index 00000000..4b17adfa --- /dev/null +++ b/src/ipa/c3-isp/algorithms/blc.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Black Level Correction control + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class Blc : public Algorithm +{ +public: + Blc() {} + ~Blc() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const c3_isp_stats_buffer *stats, + ControlList &metadata) override; + +private: + int16_t blackLevelRed_; + int16_t blackLevelGreenR_; + int16_t blackLevelGreenB_; + int16_t blackLevelBlue_; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/ccm.cpp b/src/ipa/c3-isp/algorithms/ccm.cpp new file mode 100644 index 00000000..115a5a37 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/ccm.cpp @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Color Correction Matrix control + */ + +#include "ccm.h" + +#include +#include + +#include +#include + +#include "libcamera/internal/yaml_parser.h" +#include "libipa/interpolator.h" + +/** + * \file ccm.h + */ + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +/** + * \class Ccm + * \brief A color correction matrix algorithm + */ + +LOG_DEFINE_CATEGORY(C3ISPCcm) + +Ccm::Ccm() +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Ccm::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + ccmCoeff = tuningData["CcmCoeff"].getList().value_or(std::vector{}); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Ccm::prepare([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, C3ISPParams *params) +{ + auto CcmCfg = params->block(); + CcmCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + for (unsigned int i = 0; i < 3; i++) { + for (unsigned int j = 0; j < 3; j++) { + CcmCfg->matrix[i][j] = ccmCoeff[i * 3 + j]; + } + } +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Ccm::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const c3_isp_stats_buffer *stats, + ControlList &metadata) +{ + float m[9]; + for (unsigned int i = 0; i < 3; i++) { + for (unsigned int j = 0; j < 3; j++) + m[i * 3 + j] = ccmCoeff[i * 3 + j]; + } + metadata.set(controls::ColourCorrectionMatrix, m); +} + +REGISTER_IPA_ALGORITHM(Ccm, "Ccm") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/ccm.h b/src/ipa/c3-isp/algorithms/ccm.h new file mode 100644 index 00000000..b5199136 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/ccm.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Color Correction Matrix control + */ + +#pragma once + +#include + +#include + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class Ccm : public Algorithm +{ +public: + Ccm(); + ~Ccm() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, const c3_isp_stats_buffer *stats, + ControlList &metadata) override; + +private: + std::vector ccmCoeff; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/csc.cpp b/src/ipa/c3-isp/algorithms/csc.cpp new file mode 100644 index 00000000..23d86d1e --- /dev/null +++ b/src/ipa/c3-isp/algorithms/csc.cpp @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Color Space Conversion control + */ + +#include "csc.h" + +#include +#include + +/** + * \file csc.h + */ + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +/** + * \class Csc + * \brief Color Space Conversion algorithm + */ + +LOG_DEFINE_CATEGORY(C3ISPCsc) + +Csc::Csc() +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Csc::init([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + cscCoeff = tuningData["CscCoeff"].getList().value_or(std::vector{}); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Csc::prepare([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + C3ISPParams *params) +{ + auto CscCfg = params->block(); + CscCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + for (unsigned int i = 0; i < 3; i++) { + for (unsigned int j = 0; j < 3; j++) + CscCfg->matrix[i][j] = cscCoeff[i * 3 + j]; + } +} + +REGISTER_IPA_ALGORITHM(Csc, "Csc") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/csc.h b/src/ipa/c3-isp/algorithms/csc.h new file mode 100644 index 00000000..13b89dd0 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/csc.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Color Space Conversion control + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class Csc : public Algorithm +{ +public: + Csc(); + ~Csc() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; +private: + std::vector cscCoeff; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/meson.build b/src/ipa/c3-isp/algorithms/meson.build new file mode 100644 index 00000000..5e7e76dd --- /dev/null +++ b/src/ipa/c3-isp/algorithms/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +c3isp_ipa_algorithms = files([ + 'agc.cpp', + 'awb.cpp', + 'blc.cpp', + 'ccm.cpp', + 'csc.cpp', + 'post_gamma.cpp' +]) diff --git a/src/ipa/c3-isp/algorithms/post_gamma.cpp b/src/ipa/c3-isp/algorithms/post_gamma.cpp new file mode 100644 index 00000000..3f1dcf66 --- /dev/null +++ b/src/ipa/c3-isp/algorithms/post_gamma.cpp @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Post Gamma control + */ + +#include "post_gamma.h" + +#include +#include + +/** + * \file post_gamma.h + */ + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +/** + * \class PostGamma + * \brief A post gamma algorithm + */ + +LOG_DEFINE_CATEGORY(C3ISPPostGamma) + +PostGamma::PostGamma() +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int PostGamma::init([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + gammaLut = + tuningData["GammaLut"].getList().value_or(std::vector{}); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void PostGamma::prepare([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + C3ISPParams *params) +{ + auto PostGammaCfg = params->block(); + PostGammaCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_NONE); + + for (unsigned int i = 0; i < 129; i++) { + PostGammaCfg->lut[i] = gammaLut[i]; + } +} + +REGISTER_IPA_ALGORITHM(PostGamma, "PostGamma") + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/algorithms/post_gamma.h b/src/ipa/c3-isp/algorithms/post_gamma.h new file mode 100644 index 00000000..0bfa134b --- /dev/null +++ b/src/ipa/c3-isp/algorithms/post_gamma.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP Post Gamma control + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::c3isp::algorithms { + +class PostGamma : public Algorithm +{ +public: + PostGamma(); + ~PostGamma() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + C3ISPParams *params) override; + +private: + std::vector gammaLut; +}; + +} /* namespace ipa::c3isp::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/c3-isp.cpp b/src/ipa/c3-isp/c3-isp.cpp new file mode 100644 index 00000000..02ce1b0b --- /dev/null +++ b/src/ipa/c3-isp/c3-isp.cpp @@ -0,0 +1,386 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * c3-isp.cpp - Amlogic Image Processing Algorithms + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "libcamera/internal/formats.h" +#include "libcamera/internal/mapped_framebuffer.h" +#include "libcamera/internal/yaml_parser.h" + +#include "algorithms/algorithm.h" + +#include "ipa_context.h" +#include "params.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPAC3ISP) + +using namespace std::literals::chrono_literals; + +namespace ipa::c3isp { + +static constexpr uint32_t kMaxFrameContexts = 16; + +class IPAC3ISP : public IPAC3ISPInterface, public Module +{ +public: + IPAC3ISP(); + + int init(const IPASettings &settings, unsigned int hwRevision, + const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls) override; + int start() override; + void stop() override; + + int configure(const IPAConfigInfo &ipaConfig, + ControlInfoMap *ipaControls) override; + void mapBuffers(const std::vector &buffers) override; + void unmapBuffers(const std::vector &ids) override; + + void queueRequest(const uint32_t frame, const ControlList &controls) override; + void fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) override; + void processStatsBuffer(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) override; + +protected: + std::string logPrefix() const override; + +private: + void updateControls(const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls); + void setControls(unsigned int frame); + + std::map buffers_; + std::map mappedBuffers_; + + ControlInfoMap sensorControls_; + + struct IPAContext context_; +}; + +namespace { + +const ControlInfoMap::Map c3ispControls{ + { &controls::AwbEnable, ControlInfo(false, true) }, + { &controls::ColourGains, ControlInfo(0.0f, 3.996f, 1.0f) }, +}; + +} /* namespace */ + +IPAC3ISP::IPAC3ISP() + : context_({ {}, {}, { kMaxFrameContexts }, {}, {} }) +{ +} + +std::string IPAC3ISP::logPrefix() const +{ + return "c3isp"; +} + +int IPAC3ISP::init(const IPASettings &settings, unsigned int hwRevision, + const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls) +{ + LOG(IPAC3ISP, Debug) << "Hardware revision is " << hwRevision; + + context_.camHelper = CameraSensorHelperFactoryBase::create(settings.sensorModel); + if (!context_.camHelper) { + LOG(IPAC3ISP, Error) + << "Failed to create camera sensor helper for " + << settings.sensorModel; + return -ENODEV; + } + + context_.configuration.sensor.lineDuration = + sensorInfo.minLineLength * 1.0s / sensorInfo.pixelRate; + + File file(settings.configurationFile); + if (!file.open(File::OpenModeFlag::ReadOnly)) { + int ret = file.error(); + LOG(IPAC3ISP, Error) + << "Failed to open configuration file " + << settings.configurationFile << ": " << strerror(-ret); + return ret; + } + + std::unique_ptr data = YamlParser::parse(file); + if (!data) + return -EINVAL; + + unsigned int version = (*data)["version"].get(0); + if (version != 1) { + LOG(IPAC3ISP, Error) + << "Invalid tuning file version " << version; + return -EINVAL; + } + + if (!data->contains("algorithms")) { + LOG(IPAC3ISP, Error) + << "Tuning file doesn't contain any algorithm"; + return -EINVAL; + } + + int ret = createAlgorithms(context_, (*data)["algorithms"]); + if (ret) + return ret; + + updateControls(sensorInfo, sensorControls, ipaControls); + + return 0; +} + +int IPAC3ISP::start() +{ + setControls(0); + + return 0; +} + +void IPAC3ISP::stop() +{ + context_.frameContexts.clear(); +} + +int IPAC3ISP::configure(const IPAConfigInfo &ipaConfig, + ControlInfoMap *ipaControls) +{ + sensorControls_ = ipaConfig.sensorControls; + + const auto itExp = sensorControls_.find(V4L2_CID_EXPOSURE); + int32_t minExposure = itExp->second.min().get(); + int32_t maxExposure = itExp->second.max().get(); + + const auto itGain = sensorControls_.find(V4L2_CID_ANALOGUE_GAIN); + int32_t minGain = itGain->second.min().get(); + int32_t maxGain = itGain->second.max().get(); + + LOG(IPAC3ISP, Debug) + << "Exposure: [" << minExposure << ", " << maxExposure + << "], gain: [" << minGain << ", " << maxGain << "]"; + + context_.configuration = {}; + context_.activeState = {}; + context_.frameContexts.clear(); + + const IPACameraSensorInfo &info = ipaConfig.sensorInfo; + + const ControlInfo vBlank = sensorControls_.find(V4L2_CID_VBLANK)->second; + context_.configuration.sensor.defVBlank = vBlank.def().get(); + context_.configuration.sensor.size = info.outputSize; + context_.configuration.sensor.lineDuration = info.minLineLength * 1.0s / info.pixelRate; + + updateControls(info, sensorControls_, ipaControls); + + context_.configuration.sensor.minShutterSpeed = + minExposure * context_.configuration.sensor.lineDuration; + context_.configuration.sensor.maxShutterSpeed = + maxExposure * context_.configuration.sensor.lineDuration; + context_.configuration.sensor.minAnalogueGain = + context_.camHelper->gain(minGain); + context_.configuration.sensor.maxAnalogueGain = + context_.camHelper->gain(maxGain); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast(a.get()); + + if (algo->disabled_) + continue; + + int ret = algo->configure(context_, info); + if (ret) + return ret; + } + + return 0; +} + +void IPAC3ISP::mapBuffers(const std::vector &buffers) +{ + for (const IPABuffer &buffer : buffers) { + auto elem = buffers_.emplace(std::piecewise_construct, + std::forward_as_tuple(buffer.id), + std::forward_as_tuple(buffer.planes)); + const FrameBuffer &fb = elem.first->second; + + MappedFrameBuffer mappedBuffer(&fb, MappedFrameBuffer::MapFlag::ReadWrite); + if (!mappedBuffer.isValid()) { + LOG(IPAC3ISP, Fatal) << "Failed tommap buffer: " + << strerror(mappedBuffer.error()); + } + + mappedBuffers_.emplace(buffer.id, std::move(mappedBuffer)); + } +} + +void IPAC3ISP::unmapBuffers(const std::vector &ids) +{ + for (unsigned int id : ids) { + const auto fb = buffers_.find(id); + if (fb == buffers_.end()) + continue; + + mappedBuffers_.erase(id); + buffers_.erase(id); + } +} + +void IPAC3ISP::queueRequest(const uint32_t frame, const ControlList &controls) +{ + IPAFrameContext &frameContext = context_.frameContexts.alloc(frame); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast(a.get()); + if (algo->disabled_) + continue; + algo->queueRequest(context_, frame, frameContext, controls); + } +} + +void IPAC3ISP::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + + C3ISPParams params(mappedBuffers_.at(bufferId).planes()[0]); + + for (auto const &algo : algorithms()) + algo->prepare(context_, frame, frameContext, ¶ms); + + paramsBufferReady.emit(frame, params.size()); +} + +void IPAC3ISP::processStatsBuffer(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + + const c3_isp_stats_buffer *stats = nullptr; + + stats = reinterpret_cast( + mappedBuffers_.at(bufferId).planes()[0].data()); + + frameContext.sensor.exposure = + sensorControls.get(V4L2_CID_EXPOSURE).get(); + frameContext.sensor.gain = + context_.camHelper->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + + ControlList metadata(controls::controls); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast(a.get()); + if (algo->disabled_) + continue; + algo->process(context_, frame, frameContext, stats, metadata); + } + + setControls(frame); + + metadataReady.emit(frame, metadata); +} + +void IPAC3ISP::updateControls(const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls) +{ + ControlInfoMap::Map ctrlMap = c3ispControls; + + double lineDuration = context_.configuration.sensor.lineDuration.get(); + const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second; + int32_t minExposure = v4l2Exposure.min().get() * lineDuration; + int32_t maxExposure = v4l2Exposure.max().get() * lineDuration; + int32_t defExposure = v4l2Exposure.def().get() * lineDuration; + ctrlMap.emplace(std::piecewise_construct, + std::forward_as_tuple(&controls::ExposureTime), + std::forward_as_tuple(minExposure, maxExposure, defExposure)); + + const ControlInfo &v4l2Gain = sensorControls.find(V4L2_CID_ANALOGUE_GAIN)->second; + float minGain = context_.camHelper->gain(v4l2Gain.min().get()); + float maxGain = context_.camHelper->gain(v4l2Gain.max().get()); + float defGain = context_.camHelper->gain(v4l2Gain.def().get()); + ctrlMap.emplace(std::piecewise_construct, + std::forward_as_tuple(&controls::AnalogueGain), + std::forward_as_tuple(minGain, maxGain, defGain)); + + const ControlInfo &v4l2HBlank = sensorControls.find(V4L2_CID_HBLANK)->second; + uint32_t hblank = v4l2HBlank.def().get(); + uint32_t lineLength = sensorInfo.outputSize.width + hblank; + + const ControlInfo &v4l2VBlank = sensorControls.find(V4L2_CID_VBLANK)->second; + std::array frameHeights{ + v4l2VBlank.min().get() + sensorInfo.outputSize.height, + v4l2VBlank.max().get() + sensorInfo.outputSize.height, + v4l2VBlank.def().get() + sensorInfo.outputSize.height, + }; + + std::array frameDurations; + for (unsigned int i = 0; i < frameHeights.size(); ++i) { + uint64_t frameSize = lineLength * frameHeights[i]; + frameDurations[i] = frameSize / (sensorInfo.pixelRate / 1000000U); + } + + ctrlMap[&controls::FrameDurationLimits] = ControlInfo(frameDurations[0], + frameDurations[1], + frameDurations[2]); + ctrlMap.insert(context_.ctrlMap.begin(), context_.ctrlMap.end()); + *ipaControls = ControlInfoMap(std::move(ctrlMap), controls::controls); +} + +void IPAC3ISP::setControls(unsigned int frame) +{ + uint32_t exposure = context_.activeState.agc.exposure; + uint32_t gain = context_.camHelper->gainCode(context_.activeState.agc.gain); + + ControlList ctrls(sensorControls_); + ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure)); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain)); + + setSensorControls.emit(frame, ctrls); +} + +} /* namespace ipa::c3isp */ + +/* + * External IPA module interface + */ + +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 1, + "c3isp", + "c3isp", +}; + +IPAInterface *ipaCreate() +{ + return new ipa::c3isp::IPAC3ISP(); +} +} + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/data/imx290.yaml b/src/ipa/c3-isp/data/imx290.yaml new file mode 100644 index 00000000..d2706766 --- /dev/null +++ b/src/ipa/c3-isp/data/imx290.yaml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +version: 1 +algorithms: + - Agc: + - Awb: + - PostGamma: + GammaLut: [ + 0, 86, 134, 169, 198, 223, 245, 265, 283, 300, 316, 331, + 346, 359, 373, 385, 397, 409, 420, 431, 441, 452, 462, 471, + 481, 490, 499, 508, 516, 525, 533, 541, 549, 557, 565, 572, + 580, 587, 594, 601, 608, 615, 622, 629, 635, 642, 648, 655, + 661, 667, 673, 679, 685, 691, 697, 703, 708, 714, 720, 725, + 731, 736, 742, 747, 752, 758, 763, 768, 773, 778, 783, 788, + 793, 798, 803, 808, 812, 817, 822, 827, 831, 836, 840, 845, + 849, 854, 858, 863, 867, 872, 876, 880, 884, 889, 893, 897, + 901, 905, 910, 914, 918, 922, 926, 930, 934, 938, 942, 946, + 950, 953, 957, 961, 965, 969, 972, 976, 980, 984, 987, 991, + 995, 998, 1002, 1006, 1009, 1013, 1016, 1020, 1023 + ] + - Ccm: + CcmCoeff: [ 533, -191, -86, -147, 474, -71, 23, -208, 441 ] + - Csc: + CscCoeff: [ 54, 183, 19, -29, -99, 128, 128, -116, -12 ] + - Blc: + BlcR: 20089 + BlcGr: 20090 + BlcGb: 20081 + BlcB: 20084 diff --git a/src/ipa/c3-isp/data/meson.build b/src/ipa/c3-isp/data/meson.build new file mode 100644 index 00000000..e9ecfc2c --- /dev/null +++ b/src/ipa/c3-isp/data/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +conf_files = files([ + 'imx290.yaml', +]) + +install_data(conf_files, + install_dir : ipa_data_dir / 'c3isp', + install_tag : 'runtime') diff --git a/src/ipa/c3-isp/ipa_context.cpp b/src/ipa/c3-isp/ipa_context.cpp new file mode 100644 index 00000000..3d087460 --- /dev/null +++ b/src/ipa/c3-isp/ipa_context.cpp @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * C3ISP IPA Context + */ + +#include "ipa_context.h" + +/** + * \file ipa_context.h + * \brief Context and state information shared between the algorithms + */ + +namespace libcamera::ipa::c3isp { + +/** + * \struct IPASessionConfiguration + * \brief Session configuration for the IPA module + * + * The session configuration contains all IPA configuration parameters that + * remain constant during the capture session, from IPA module start to stop. + * It is typically set during the configure() operation of the IPA module, but + * may also be updated in the start() operation. + */ + +/** + * \var IPASessionConfiguration::sensor + * \brief Sensor-specific configuration of the IPA + * + * \var IPASessionConfiguration::sensor.minShutterSpeed + * \brief Minimum shutter speed supported with the sensor + * + * \var IPASessionConfiguration::sensor.maxShutterSpeed + * \brief Maximum shutter speed supported with the sensor + * + * \var IPASessionConfiguration::sensor.minAnalogueGain + * \brief Minimum analogue gain supported with the sensor + * + * \var IPASessionConfiguration::sensor.maxAnalogueGain + * \brief Maximum analogue gain supported with the sensor + * + * \var IPASessionConfiguration::sensor.defVBlank + * \brief The default vblank value of the sensor + * + * \var IPASessionConfiguration::sensor.lineDuration + * \brief Line duration in microseconds + * + * \var IPASessionConfiguration::sensor.size + * \brief Sensor output resolution + */ + +/** + * \struct IPAActiveState + * \brief Active state for algorithms + * + * The active state contains all algorithm-specific data that needs to be + * maintained by algorithms across frames. Unlike the session configuration, + * the active state is mutable and constantly updated by algorithms. The active + * state is accessible through the IPAContext structure. + * + * The active state stores two distinct categories of information: + * + * - The consolidated value of all algorithm controls. Requests passed to + * the queueRequest() function store values for controls that the + * application wants to modify for that particular frame, and the + * queueRequest() function updates the active state with those values. + * The active state thus contains a consolidated view of the value of all + * controls handled by the algorithm. + * + * - The value of parameters computed by the algorithm when running in auto + * mode. Algorithms running in auto mode compute new parameters every + * time statistics buffers are received (either synchronously, or + * possibly in a background thread). The latest computed value of those + * parameters is stored in the active state in the process() function. + * + * Each of the members in the active state belongs to a specific algorithm. A + * member may be read by any algorithm, but shall only be written by its owner. + */ + +/** + * \var IPAActiveState::agc + * \brief State for the Automatic Gain Control algorithm + * + * The \a automatic variables track the latest values computed by algorithm + * based on the latest processed statistics. All other variables track the + * consolidated controls requested in queued requests. + * + * \var IPAActiveState::agc.exposure + * \brief Automatic exposure time expressed as a number of lines + * + * \var IPAActiveState::agc.gain + * \brief Automatic analogue gain multiplier + * + * \var IPAActiveState::agc.constraintMode + * \brief Constraint mode as set by the AeConstraintMode control + * + * \var IPAActiveState::agc.exposureMode + * \brief Exposure mode as set by the AeExposureMode control + */ + +/** + * \var IPAActiveState::awb + * \brief State for the Automatic White Balance algorithm + * + * \struct IPAActiveState::awb.gains + * \brief White balance gains + * + * \struct IPAActiveState::awb.gains.manual + * \brief Manual white balance gains (set through requests) + * + * \var IPAActiveState::awb.gains.manual.red + * \brief Manual white balance gain for R channel + * + * \var IPAActiveState::awb.gains.manual.green + * \brief Manual white balance gain for G channel + * + * \var IPAActiveState::awb.gains.manual.blue + * \brief Manual white balance gain for B channel + * + * \struct IPAActiveState::awb.gains.automatic + * \brief Automatic white balance gains (computed by the algorithm) + * + * \var IPAActiveState::awb.gains.automatic.red + * \brief Automatic white balance gain for R channel + * + * \var IPAActiveState::awb.gains.automatic.green + * \brief Automatic white balance gain for G channel + * + * \var IPAActiveState::awb.gains.automatic.blue + * \brief Automatic white balance gain for B channel + * + * \var IPAActiveState::awb.temperatureK + * \brief Estimated color temperature + * + * \var IPAActiveState::awb.autoEnabled + * \brief Whether the Auto White Balance algorithm is enabled + */ + +/** + * \struct IPAFrameContext + * \brief Per-frame context for algorithms + * + * The frame context stores two distinct categories of information: + * + * - The value of the controls to be applied to the frame. These values are + * typically set in the queueRequest() function, from the consolidated + * control values stored in the active state. The frame context thus stores + * values for all controls related to the algorithm, not limited to the + * controls specified in the corresponding request, but consolidated from all + * requests that have been queued so far. + * + * For controls that can be set manually or computed by an algorithm + * (depending on the algorithm operation mode), such as for instance the + * colour gains for the AWB algorithm, the control value will be stored in + * the frame context in the queueRequest() function only when operating in + * manual mode. When operating in auto mode, the values are computed by the + * algorithm in process(), stored in the active state, and copied to the + * frame context in prepare(), just before being stored in the ISP parameters + * buffer. + * + * The queueRequest() function can also store ancillary data in the frame + * context, such as flags to indicate if (and what) control values have + * changed compared to the previous request. + * + * - Status information computed by the algorithm for a frame. For instance, + * the colour temperature estimated by the AWB algorithm from ISP statistics + * calculated on a frame is stored in the frame context for that frame in + * the process() function. + */ + +/** + * \var IPAFrameContext::agc + * \brief Automatic Gain Control parameters for this frame + * + * The exposure and gain are provided by the AGC algorithm, and are to be + * applied to the sensor in order to take effect for this frame. + * + * \var IPAFrameContext::agc.exposure + * \brief Exposure time expressed as a number of lines computed by the algorithm + * + * \var IPAFrameContext::agc.gain + * \brief Analogue gain multiplier computed by the algorithm + * + * The gain should be adapted to the sensor specific gain code before applying. + * + * \var IPAFrameContext::agc.autoEnabled + * \brief Manual/automatic AGC state as set by the AeEnable control + * + * \var IPAFrameContext::agc.constraintMode + * \brief Constraint mode as set by the AeConstraintMode control + * + * \var IPAFrameContext::agc.exposureMode + * \brief Exposure mode as set by the AeExposureMode control + * + * \var IPAFrameContext::agc.meteringMode + * \brief Metering mode as set by the AeMeteringMode control + * + * \var IPAFrameContext::agc.maxFrameDuration + * \brief Maximum frame duration as set by the FrameDurationLimits control + * + * \var IPAFrameContext::agc.updateMetering + * \brief Indicate if new ISP AGC metering parameters need to be applied + */ + +/** + * \var IPAFrameContext::awb + * \brief Automatic White Balance parameters for this frame + * + * \struct IPAFrameContext::awb.gains + * \brief White balance gains + * + * \var IPAFrameContext::awb.gains.red + * \brief White balance gain for R channel + * + * \var IPAFrameContext::awb.gains.green + * \brief White balance gain for G channel + * + * \var IPAFrameContext::awb.gains.blue + * \brief White balance gain for B channel + * + * \var IPAFrameContext::awb.autoEnabled + * \brief Whether the Auto White Balance algorithm is enabled + */ + +/** + * \var IPAFrameContext::sensor + * \brief Sensor configuration that used been used for this frame + * + * \var IPAFrameContext::sensor.exposure + * \brief Exposure time expressed as a number of lines + * + * \var IPAFrameContext::sensor.gain + * \brief Analogue gain multiplier + */ + +/** + * \struct IPAContext + * \brief Global IPA context data shared between all algorithms + * + * \var IPAContext::configuration + * \brief The IPA session configuration, immutable during the session + * + * \var IPAContext::activeState + * \brief The IPA active state, storing the latest state for all algorithms + * + * \var IPAContext::frameContexts + * \brief Ring buffer of per-frame contexts + */ + +} /* namespace libcamera::ipa::c3isp */ diff --git a/src/ipa/c3-isp/ipa_context.h b/src/ipa/c3-isp/ipa_context.h new file mode 100644 index 00000000..f1519c4c --- /dev/null +++ b/src/ipa/c3-isp/ipa_context.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * C3ISP IPA Context + */ + +#pragma once + +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +namespace ipa::c3isp { + +struct IPASessionConfiguration { + struct { + utils::Duration minShutterSpeed; + utils::Duration maxShutterSpeed; + double minAnalogueGain; + double maxAnalogueGain; + + int32_t defVBlank; + utils::Duration lineDuration; + Size size; + } sensor; +}; + +struct IPAActiveState { + struct { + uint32_t exposure; + double gain; + uint32_t constraintMode; + uint32_t exposureMode; + } agc; + + struct { + struct { + struct { + double red; + double green; + double blue; + } manual; + struct { + double red; + double green; + double blue; + } automatic; + } gains; + + unsigned int temperatureK; + bool autoEnabled; + } awb; +}; + +struct IPAFrameContext : public FrameContext { + struct { + uint32_t exposure; + double gain; + bool autoEnabled; + controls::AeConstraintModeEnum constraintMode; + controls::AeExposureModeEnum exposureMode; + controls::AeMeteringModeEnum meteringMode; + utils::Duration maxFrameDuration; + bool updateMetering; + } agc; + + struct { + struct { + double red; + double green; + double blue; + } gains; + + bool autoEnabled; + } awb; + + struct { + uint32_t exposure; + double gain; + } sensor; +}; + +struct IPAContext { + IPASessionConfiguration configuration; + IPAActiveState activeState; + + FCQueue frameContexts; + + ControlInfoMap::Map ctrlMap; + + /* Interface to the Camera Helper */ + std::unique_ptr camHelper; +}; + +} /* namespace ipa::c3isp */ + +} /* namespace libcamera*/ diff --git a/src/ipa/c3-isp/meson.build b/src/ipa/c3-isp/meson.build new file mode 100644 index 00000000..fa5c6be0 --- /dev/null +++ b/src/ipa/c3-isp/meson.build @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: CC0-1.0 + +subdir('algorithms') +subdir('data') + +ipa_name = 'ipa_c3isp' + +c3isp_ipa_sources = files([ + 'ipa_context.cpp', + 'params.cpp', + 'c3-isp.cpp', +]) + +c3isp_ipa_sources += c3isp_ipa_algorithms + +mod = shared_module(ipa_name, c3isp_ipa_sources, + name_prefix : '', + include_directories : [ipa_includes], + dependencies : [libcamera_private, libipa_dep], + install : true, + install_dir : ipa_install_dir) + +if ipa_sign_module + custom_target(ipa_name + '.so.sign', + input : mod, + output : ipa_name + '.so.sign', + command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'], + install : false, + build_by_default : true) +endif + +ipa_names += ipa_name diff --git a/src/ipa/c3-isp/module.h b/src/ipa/c3-isp/module.h new file mode 100644 index 00000000..07116f80 --- /dev/null +++ b/src/ipa/c3-isp/module.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic + * + * C3ISP IPA Module + */ + +#pragma once + +#include + +#include + +#include + +#include "ipa_context.h" +#include "params.h" + +namespace libcamera { + +namespace ipa::c3isp { + +using Module = ipa::Module; + +} /* namespace ipa::c3isp */ + +} /* namespace libcamera*/ diff --git a/src/ipa/c3-isp/params.cpp b/src/ipa/c3-isp/params.cpp new file mode 100644 index 00000000..fd682b53 --- /dev/null +++ b/src/ipa/c3-isp/params.cpp @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * C3ISP ISP Parameters + */ + +#include "params.h" + +#include +#include +#include + +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(C3ISPParams) + +namespace ipa::c3isp { + +namespace { + +struct BlockTypeInfo { + enum c3_isp_params_block_type type; + size_t size; +}; + +#define C3ISP_BLOCK_TYPE_ENTRY(block, id, type) \ + { \ + BlockType::block, \ + { \ + C3_ISP_PARAMS_BLOCK_##id, \ + sizeof(struct c3_isp_params_##type), \ + } \ + } + +const std::map kBlockTypeInfo = { + C3ISP_BLOCK_TYPE_ENTRY(AWBGains, AWB_GAINS, awb_gains), + C3ISP_BLOCK_TYPE_ENTRY(AWBConfig, AWB_CONFIG, awb_config), + C3ISP_BLOCK_TYPE_ENTRY(AEConfig, AE_CONFIG, ae_config), + C3ISP_BLOCK_TYPE_ENTRY(AFConfig, AF_CONFIG, af_config), + C3ISP_BLOCK_TYPE_ENTRY(PostGamma, PST_GAMMA, pst_gamma), + C3ISP_BLOCK_TYPE_ENTRY(Ccm, CCM, ccm), + C3ISP_BLOCK_TYPE_ENTRY(Csc, CSC, csc), + C3ISP_BLOCK_TYPE_ENTRY(Blc, BLC, blc), +}; + +} /* namespace */ + +C3ISPParamsBlockBase::C3ISPParamsBlockBase(BlockType type, + const Span &data) + : type_(type), data_(data) +{ + header_ = data.subspan(0, sizeof(c3_isp_params_block_header)); +} + +void C3ISPParamsBlockBase::setEnabled(uint16_t flags) +{ + struct c3_isp_params_block_header *header = + reinterpret_cast(header_.data()); + + header->flags = flags; +} + +C3ISPParams::C3ISPParams(Span data) + : data_(data), used_(0) +{ + struct c3_isp_params_cfg *buffer = + reinterpret_cast(data.data()); + + buffer->version = C3_ISP_PARAMS_BUFFER_V0; + buffer->data_size = 0; + + used_ += offsetof(struct c3_isp_params_cfg, data); +} + +Span C3ISPParams::block(BlockType type) +{ + auto infoIt = kBlockTypeInfo.find(type); + if (infoIt == kBlockTypeInfo.end()) { + LOG(C3ISPParams, Error) + << "Invalid parameters type " + << utils::to_underlying(type); + return {}; + } + + const BlockTypeInfo &info = infoIt->second; + + auto cacheIt = blocks_.find(type); + if (cacheIt != blocks_.end()) + return cacheIt->second; + + size_t size = info.size; + if (size > data_.size() - used_) { + LOG(C3ISPParams, Error) + << "Out of memory to allocate block type " + << utils::to_underlying(type); + return {}; + } + + Span block = data_.subspan(used_, info.size); + used_ += block.size(); + + struct c3_isp_params_cfg *buffer = + reinterpret_cast(data_.data()); + buffer->data_size += block.size(); + + memset(block.data(), 0, block.size()); + + struct c3_isp_params_block_header *header = + reinterpret_cast(block.data()); + header->type = info.type; + header->size = block.size(); + + blocks_[type] = block; + + return block; +} + +} /* namespace ipa::c3isp */ + +} /* namespace libcamera */ diff --git a/src/ipa/c3-isp/params.h b/src/ipa/c3-isp/params.h new file mode 100644 index 00000000..9bb3877b --- /dev/null +++ b/src/ipa/c3-isp/params.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * C3ISP ISP Parameters + */ + +#pragma once + +#include +#include + +#include + +#include +#include + +namespace libcamera { + +namespace ipa::c3isp { + +enum class BlockType { + AWBGains, + AWBConfig, + AEConfig, + AFConfig, + PostGamma, + Ccm, + Csc, + Blc, +}; + +namespace details { + +template +struct block_type { +}; + +#define C3ISP_DEFINE_BLOCK_TYPE(blocktype, blockStruct) \ + template<> \ + struct block_type { \ + using type = struct c3_isp_params_##blockStruct; \ + }; + +C3ISP_DEFINE_BLOCK_TYPE(AWBGains, awb_gains) +C3ISP_DEFINE_BLOCK_TYPE(AWBConfig, awb_config) +C3ISP_DEFINE_BLOCK_TYPE(AEConfig, ae_config) +C3ISP_DEFINE_BLOCK_TYPE(AFConfig, af_config) +C3ISP_DEFINE_BLOCK_TYPE(PostGamma, pst_gamma) +C3ISP_DEFINE_BLOCK_TYPE(Ccm, ccm) +C3ISP_DEFINE_BLOCK_TYPE(Csc, csc) +C3ISP_DEFINE_BLOCK_TYPE(Blc, blc) + +} /* namespace details */ + +class C3ISPParams; + +class C3ISPParamsBlockBase +{ +public: + C3ISPParamsBlockBase(BlockType type, const Span &data); + + Span data() const { return data_; } + + void setEnabled(uint16_t flags); + +private: + LIBCAMERA_DISABLE_COPY(C3ISPParamsBlockBase) + + BlockType type_; + Span header_; + Span data_; +}; + +template +class C3ISPParamsBlock : public C3ISPParamsBlockBase +{ +public: + using Type = typename details::block_type::type; + + C3ISPParamsBlock(const Span &data) + : C3ISPParamsBlockBase(B, data) + { + } + + const Type *operator->() const + { + return reinterpret_cast(data().data()); + } + + Type *operator->() + { + return reinterpret_cast(data().data()); + } + + const Type &operator*() const & + { + return *reinterpret_cast(data().data()); + } + + const Type &operator*() & + { + return *reinterpret_cast(data().data()); + } +}; + +class C3ISPParams +{ +public: + C3ISPParams(Span data); + + template + C3ISPParamsBlock block() + { + return C3ISPParamsBlock(block(B)); + } + + size_t size() const { return used_; } + +private: + friend class C3ISPParamsBlockBase; + + Span block(BlockType type); + + Span data_; + size_t used_; + + std::map> blocks_; +}; + +} /* namespace ipa::c3isp */ + +} /* namespace libcamera */