From patchwork Thu Feb 27 10:57:24 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Keke Li X-Patchwork-Id: 22885 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 4CB40BF415 for ; Thu, 27 Feb 2025 10:57:48 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 8DEBB6875B; Thu, 27 Feb 2025 11:57:46 +0100 (CET) Received: from mail-sh.amlogic.com (unknown [58.32.228.46]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id EEE366874F for ; Thu, 27 Feb 2025 11:57:40 +0100 (CET) Received: from droid10.amlogic.com (10.18.11.213) by mail-sh.amlogic.com (10.18.11.5) with Microsoft SMTP Server id 15.1.2507.39; Thu, 27 Feb 2025 18:57:37 +0800 From: Keke Li To: CC: , , , Keke Li Subject: [PATCH v3 02/11] libcamera: pipeline: Add c3-isp pipeline handler Date: Thu, 27 Feb 2025 18:57:24 +0800 Message-ID: <20250227105733.187611-3-keke.li@amlogic.com> X-Mailer: git-send-email 2.29.0 In-Reply-To: <20250227105733.187611-1-keke.li@amlogic.com> References: <20250227105733.187611-1-keke.li@amlogic.com> MIME-Version: 1.0 X-Originating-IP: [10.18.11.213] X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The Amlogic C3 ISP pipeline handler supports 3-channel image output, 1-channel 3A statistical information ouput and 1-channel parameters input. Signed-off-by: Keke Li --- Documentation/Doxyfile-common.in | 1 + include/libcamera/ipa/c3-isp.mojom | 34 + include/libcamera/ipa/meson.build | 1 + meson_options.txt | 1 + src/libcamera/pipeline/c3-isp/c3-isp.cpp | 1313 +++++++++++++++++++++ src/libcamera/pipeline/c3-isp/meson.build | 5 + 6 files changed, 1355 insertions(+) create mode 100644 include/libcamera/ipa/c3-isp.mojom create mode 100644 src/libcamera/pipeline/c3-isp/c3-isp.cpp create mode 100644 src/libcamera/pipeline/c3-isp/meson.build diff --git a/Documentation/Doxyfile-common.in b/Documentation/Doxyfile-common.in index 045c19dd..19f28886 100644 --- a/Documentation/Doxyfile-common.in +++ b/Documentation/Doxyfile-common.in @@ -31,6 +31,7 @@ RECURSIVE = YES EXCLUDE_PATTERNS = @TOP_BUILDDIR@/include/libcamera/ipa/*_serializer.h \ @TOP_BUILDDIR@/include/libcamera/ipa/*_proxy.h \ + @TOP_BUILDDIR@/include/libcamera/ipa/c3-isp_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/ipu3_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/mali-c55_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/raspberrypi_*.h \ diff --git a/include/libcamera/ipa/c3-isp.mojom b/include/libcamera/ipa/c3-isp.mojom new file mode 100644 index 00000000..f8556256 --- /dev/null +++ b/include/libcamera/ipa/c3-isp.mojom @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +module ipa.c3isp; + +import "include/libcamera/ipa/core.mojom"; + +struct IPAConfigInfo { + libcamera.IPACameraSensorInfo sensorInfo; + libcamera.ControlInfoMap sensorControls; +}; + +interface IPAC3ISPInterface { + init(libcamera.IPASettings settings, IPAConfigInfo configInfo) + => (int32 ret, libcamera.ControlInfoMap ipaControls); + start() => (int32 ret); + stop(); + + configure(IPAConfigInfo configInfo) + => (int32 ret, libcamera.ControlInfoMap ipaControls); + + mapBuffers(array buffers, bool readOnly); + unmapBuffers(array buffers); + + [async] queueRequest(uint32 request, libcamera.ControlList reqControls); + [async] computeParams(uint32 request, uint32 bufferId); + [async] processStats(uint32 request, uint32 bufferId, + libcamera.ControlList sensorControls); +}; + +interface IPAC3ISPEventInterface { + paramsComputed(uint32 request, uint32 bytesused); + statsProcessed(uint32 request, libcamera.ControlList metadata); + setSensorControls(libcamera.ControlList sensorControls); +}; diff --git a/include/libcamera/ipa/meson.build b/include/libcamera/ipa/meson.build index 3129f119..bc0fa7d3 100644 --- a/include/libcamera/ipa/meson.build +++ b/include/libcamera/ipa/meson.build @@ -63,6 +63,7 @@ libcamera_ipa_headers += custom_target('core_ipa_serializer_h', # Mapping from pipeline handler name to mojom file pipeline_ipa_mojom_mapping = { + 'c3-isp': 'c3-isp.mojom', 'ipu3': 'ipu3.mojom', 'mali-c55': 'mali-c55.mojom', 'rkisp1': 'rkisp1.mojom', diff --git a/meson_options.txt b/meson_options.txt index f19bca91..998a4463 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -46,6 +46,7 @@ option('pipelines', choices : [ 'all', 'auto', + 'c3-isp', 'imx8-isi', 'ipu3', 'mali-c55', diff --git a/src/libcamera/pipeline/c3-isp/c3-isp.cpp b/src/libcamera/pipeline/c3-isp/c3-isp.cpp new file mode 100644 index 00000000..90baedc8 --- /dev/null +++ b/src/libcamera/pipeline/c3-isp/c3-isp.cpp @@ -0,0 +1,1313 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Amlogic Inc. + * + * Pipeline Handler for Amlogic C3 ISP + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/delayed_controls.h" +#include "libcamera/internal/device_enumerator.h" +#include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/ipa_manager.h" +#include "libcamera/internal/media_device.h" +#include "libcamera/internal/pipeline_handler.h" +#include "libcamera/internal/request.h" +#include "libcamera/internal/v4l2_subdevice.h" +#include "libcamera/internal/v4l2_videodevice.h" + +namespace { + +bool isFmtRaw(const libcamera::PixelFormat &pixFmt) +{ + return libcamera::PixelFormatInfo::info(pixFmt).colourEncoding == + libcamera::PixelFormatInfo::ColourEncodingRAW; +} + +} /* namespace */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(C3ISP) + +const std::map C3ISPFmtToCode = { + { formats::R8, MEDIA_BUS_FMT_YUV10_1X30 }, + { formats::NV12, MEDIA_BUS_FMT_YUV10_1X30 }, + { formats::NV21, MEDIA_BUS_FMT_YUV10_1X30 }, + { formats::NV16, MEDIA_BUS_FMT_YUV10_1X30 }, + { formats::NV61, MEDIA_BUS_FMT_YUV10_1X30 }, + + /* RAW formats, STILL pipe only. */ + { formats::SRGGB12, MEDIA_BUS_FMT_SRGGB16_1X16 }, + { formats::SBGGR12, MEDIA_BUS_FMT_SBGGR16_1X16 }, + { formats::SGRBG12, MEDIA_BUS_FMT_SGRBG16_1X16 }, + { formats::SGBRG12, MEDIA_BUS_FMT_SGBRG16_1X16 }, +}; + +constexpr Size kC3ISPMinSize = { 160, 120 }; +constexpr Size kC3ISPMaxSize = { 2888, 2240 }; + +struct C3ISPFrameInfo { + Request *request; + + FrameBuffer *paramBuffer; + FrameBuffer *statBuffer; + + bool paramsDone; + bool statsDone; +}; + +class C3ISPCameraData : public Camera::Private +{ +public: + C3ISPCameraData(PipelineHandler *pipe, MediaEntity *entity) + : Camera::Private(pipe), entity_(entity) + { + } + + int init(); + int loadIPA(); + + int pixfmtToMbusCode(const PixelFormat &pixFmt) const; + const PixelFormat &bestRawFmt() const; + + PixelFormat adjustRawFmt(const PixelFormat &pixFmt) const; + Size adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const; + + void updateControls(const ControlInfoMap &ipaControls); + + MediaEntity *entity_; + std::unique_ptr sensor_; + std::unique_ptr csi_; + std::unique_ptr adap_; + Stream viewStream; + Stream stillStream; + Stream videoStream; + + std::unique_ptr ipa_; + std::vector ipaStatBuffers_; + std::vector ipaParamBuffers_; + + std::unique_ptr delayedCtrls_; + +private: + void setSensorControls(const ControlList &sensorControls); +}; + +int C3ISPCameraData::init() +{ + /* Register a CameraSensor */ + sensor_ = CameraSensorFactoryBase::create(entity_); + if (!sensor_) + return -EINVAL; + + const MediaPad *sensorSrc = entity_->getPadByIndex(0); + MediaEntity *csiEntity = sensorSrc->links()[0]->sink()->entity(); + + csi_ = std::make_unique(csiEntity); + if (csi_->open()) { + LOG(C3ISP, Error) << "Failed to open CSI-2 subdevice"; + return -EINVAL; + } + + const MediaPad *csiSrc = csiEntity->getPadByIndex(1); + MediaEntity *adapEntity = csiSrc->links()[0]->sink()->entity(); + + adap_ = std::make_unique(adapEntity); + if (adap_->open()) { + LOG(C3ISP, Error) << "Failed to open adapter subdevice"; + return -EINVAL; + } + + return 0; +} + +void C3ISPCameraData::setSensorControls(const ControlList &sensorControls) +{ + delayedCtrls_->push(sensorControls); +} + +/* + * Search the sensor's supported formats for the one with a matching bayer + * order and the greatest bitdepth. + */ +int C3ISPCameraData::pixfmtToMbusCode(const PixelFormat &pixFmt) const +{ + auto it = C3ISPFmtToCode.find(pixFmt); + if (it == C3ISPFmtToCode.end()) + return -EINVAL; + + BayerFormat bayerFmt = BayerFormat::fromMbusCode(it->second); + if (!bayerFmt.isValid()) + return -EINVAL; + + unsigned int snsMbusCode = 0; + unsigned int bitDepth = 0; + for (const auto &code : sensor_->mbusCodes()) { + BayerFormat snsBayerFmt = BayerFormat::fromMbusCode(code); + if (!snsBayerFmt.isValid()) + continue; + + if (snsBayerFmt.order != bayerFmt.order) + continue; + + if (snsBayerFmt.bitDepth > bitDepth) { + bitDepth = snsBayerFmt.bitDepth; + snsMbusCode = code; + } + } + + if (!snsMbusCode) + return -EINVAL; + + return snsMbusCode; +} + +/* Find a RAW PixelFormat supported by both the ISP and the sensor. */ +const PixelFormat &C3ISPCameraData::bestRawFmt() const +{ + static const PixelFormat invalidPixFmt = {}; + + for (const auto &code : sensor_->mbusCodes()) { + BayerFormat sensorBayer = BayerFormat::fromMbusCode(code); + + if (!sensorBayer.isValid()) + continue; + + for (const auto &[pixFmt, rawCode] : C3ISPFmtToCode) { + if (!isFmtRaw(pixFmt)) + continue; + + BayerFormat bayer = BayerFormat::fromMbusCode(rawCode); + if (bayer.order == sensorBayer.order) + return pixFmt; + } + } + + LOG(C3ISP, Error) << "Can't get a compatible format"; + + return invalidPixFmt; +} + +/* + * Make sure the provided raw pixel format is supported and adjust it to + * one of the supported ones if it's not. + */ +PixelFormat C3ISPCameraData::adjustRawFmt(const PixelFormat &rawFmt) const +{ + int rawCode = pixfmtToMbusCode(rawFmt); + if (rawCode < 0) + return bestRawFmt(); + + const auto rawSizes = sensor_->sizes(rawCode); + if (rawSizes.empty()) + return bestRawFmt(); + + return rawFmt; +} + +Size C3ISPCameraData::adjustRawSizes(const PixelFormat &rawFmt, const Size &size) const +{ + Size rawSize = size.boundedTo(kC3ISPMaxSize); + + int rawCode = pixfmtToMbusCode(rawFmt); + if (rawCode < 0) + return {}; + + const auto rawSizes = sensor_->sizes(rawCode); + + auto sizeIt = std::find(rawSizes.begin(), rawSizes.end(), rawSize); + if (sizeIt != rawSizes.end()) + return rawSize; + + /* Adjust the rawSize to the closest supported size */ + uint16_t distance = std::numeric_limits::max(); + Size bestSize; + for (const Size &sz : rawSizes) { + uint16_t dist = std::abs(static_cast(rawSize.width) - + static_cast(sz.width)) + + std::abs(static_cast(rawSize.height) - + static_cast(sz.height)); + if (dist < distance) { + distance = dist; + bestSize = sz; + } + } + + return bestSize; +} + +void C3ISPCameraData::updateControls(const ControlInfoMap &ipaControls) +{ + ControlInfoMap::Map controls; + + for (auto const &c : ipaControls) + controls.emplace(c.first, c.second); + + controlInfo_ = ControlInfoMap(std::move(controls), controls::controls); +} + +int C3ISPCameraData::loadIPA() +{ + int ret; + + ipa_ = IPAManager::createIPA(pipe(), 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->setSensorControls.connect(this, &C3ISPCameraData::setSensorControls); + + std::string ipaTuningFile = ipa_->configurationFile(sensor_->model() + ".yaml", + "uncalibrated.yaml"); + + ipa::c3isp::IPAConfigInfo ipaConfig{}; + + ret = sensor_->sensorInfo(&ipaConfig.sensorInfo); + if (ret) + return ret; + + ipaConfig.sensorControls = sensor_->controls(); + + ControlInfoMap ipaControls; + ret = ipa_->init({ ipaTuningFile, sensor_->model() }, ipaConfig, &ipaControls); + if (ret) { + LOG(C3ISP, Error) << "Failed to initialize IPA"; + return ret; + } + + updateControls(ipaControls); + + return 0; +} + +class C3ISPCameraConfiguration : public CameraConfiguration +{ +public: + C3ISPCameraConfiguration(C3ISPCameraData *data) + : CameraConfiguration(), data_(data) + { + } + + Status validate() override; + const Transform &combinedTransform() { return combinedTransform_; } + + V4L2SubdeviceFormat sensorFormat_; + +private: + static constexpr unsigned int kMaxStreams = 3; + + const C3ISPCameraData *data_; + Transform combinedTransform_; +}; + +CameraConfiguration::Status C3ISPCameraConfiguration::validate() +{ + Status status = Valid; + + if (config_.empty()) + return Invalid; + + if (config_.size() > kMaxStreams) { + config_.resize(kMaxStreams); + status = Adjusted; + } + + Orientation requestOrientation = orientation; + combinedTransform_ = data_->sensor_->computeTransform(&orientation); + if (orientation != requestOrientation) + status = Adjusted; + + bool stillPipeAvailable = true; + StreamConfiguration *rawConfig = nullptr; + for (StreamConfiguration &config : config_) { + if (!isFmtRaw(config.pixelFormat)) + continue; + + if (rawConfig) { + LOG(C3ISP, Error) << "Only support a RAW stream"; + return Invalid; + } + + rawConfig = &config; + } + + /* + * The C3 ISP can not upscale. Limit the configuration to the ISP + * capabilities and the sensor resolution. + */ + Size maxSize = kC3ISPMaxSize.boundedTo(data_->sensor_->resolution()); + if (rawConfig) { + PixelFormat rawFmt = + data_->adjustRawFmt(rawConfig->pixelFormat); + + if (!rawFmt.isValid()) + return Invalid; + + if (rawFmt != rawConfig->pixelFormat) { + LOG(C3ISP, Debug) + << "Adjust RAW format to " << rawFmt; + rawConfig->pixelFormat = rawFmt; + status = Adjusted; + } + + Size rawSize = data_->adjustRawSizes(rawFmt, rawConfig->size); + if (rawSize != rawConfig->size) { + LOG(C3ISP, Debug) + << "Adjust RAW size to " << rawSize; + rawConfig->size = rawSize; + status = Adjusted; + } + + maxSize = rawSize; + + const PixelFormatInfo &info = PixelFormatInfo::info(rawConfig->pixelFormat); + rawConfig->stride = info.stride(rawConfig->size.width, 0, 16); + rawConfig->frameSize = info.frameSize(rawConfig->size, 4); + + rawConfig->setStream(const_cast(&data_->stillStream)); + stillPipeAvailable = false; + } + + /* Adjust processed streams */ + + bool videoPipeAvailable = true; + Size maxConfigSize; + for (StreamConfiguration &config : config_) { + if (isFmtRaw(config.pixelFormat)) + continue; + + const auto it = C3ISPFmtToCode.find(config.pixelFormat); + if (it == C3ISPFmtToCode.end()) { + LOG(C3ISP, Debug) + << "Format adjusted to " << formats::NV12; + config.pixelFormat = formats::NV12; + status = Adjusted; + } + + Size size = std::clamp(config.size, kC3ISPMinSize, maxSize); + if (size != config.size) { + LOG(C3ISP, Debug) + << "Size adjusted to " << size; + config.size = size; + status = Adjusted; + } + + if (maxConfigSize < size) + maxConfigSize = size; + + if (stillPipeAvailable) { + config.setStream(const_cast(&data_->stillStream)); + stillPipeAvailable = false; + } else if (videoPipeAvailable) { + config.setStream(const_cast(&data_->videoStream)); + videoPipeAvailable = false; + } else { + config.setStream(const_cast(&data_->viewStream)); + } + } + + /* Configure the sensor format */ + + /* If there's a RAW config, sensor format follow it */ + if (rawConfig) { + sensorFormat_.code = data_->pixfmtToMbusCode(rawConfig->pixelFormat); + sensorFormat_.size = rawConfig->size; + + LOG(C3ISP, Debug) << "RAW configuration format " << sensorFormat_; + + return status; + } + + /* If there's no RAW config, compture the sensor format */ + PixelFormat rawFormat = data_->bestRawFmt(); + if (!rawFormat.isValid()) + return Invalid; + + sensorFormat_.code = data_->pixfmtToMbusCode(rawFormat); + sensorFormat_.size = data_->adjustRawSizes(rawFormat, maxConfigSize); + + LOG(C3ISP, Debug) << "Sensor format " << sensorFormat_; + + return status; +} + +class PipelineHandlerC3ISP : public PipelineHandler +{ +public: + PipelineHandlerC3ISP(CameraManager *manager); + + std::unique_ptr generateConfiguration(Camera *camera, + Span roles) override; + int configure(Camera *camera, CameraConfiguration *config) override; + + int exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) override; + + int allocateBuffers(Camera *camera); + void freeBuffers(Camera *camera); + + int start(Camera *camera, const ControlList *controls) override; + void stopDevice(Camera *camera) override; + + int queueRequestDevice(Camera *camera, Request *request) override; + + void imageBufferReady(FrameBuffer *buffer); + void paramsBufferReady(FrameBuffer *buffer); + void statsBufferReady(FrameBuffer *buffer); + void paramsComputed(unsigned int requestId, unsigned int bytesused); + void statsProcessed(unsigned int requestId, const ControlList &metadata); + + bool match(DeviceEnumerator *enumerator) override; + +private: + struct C3ISPPipe { + std::unique_ptr resizer; + std::unique_ptr cap; + Stream *stream; + }; + + enum { + C3ISPVIEW, + C3ISPVIDEO, + C3ISPSTILL, + C3ISPNumPipes, + }; + + C3ISPCameraData *cameraData(Camera *camera) + { + return static_cast(camera->_d()); + } + + C3ISPPipe *pipeFromStream(C3ISPCameraData *data, Stream *stream) + { + if (stream == &data->viewStream) + return &pipes_[C3ISPVIEW]; + else if (stream == &data->stillStream) + return &pipes_[C3ISPSTILL]; + else if (stream == &data->videoStream) + return &pipes_[C3ISPVIDEO]; + else + LOG(C3ISP, Fatal) << "Invalid stream: " << stream; + + return nullptr; + } + + C3ISPPipe *pipeFromStream(C3ISPCameraData *data, const Stream *stream) + { + return pipeFromStream(data, const_cast(stream)); + } + + void resetPipes() + { + for (C3ISPPipe &pipe : pipes_) + pipe.stream = nullptr; + } + + int pipesStart(); + void pipesStop(); + + int configureRawStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat); + int configureProcessedStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat); + + bool createCamera(MediaLink *ispLink); + void tryComplete(C3ISPFrameInfo *info); + C3ISPFrameInfo *findFrameInfo(FrameBuffer *buffer); + C3ISPFrameInfo *findFrameInfo(Request *request); + void clearIncompleteRequests(); + + MediaDevice *media_; + std::unique_ptr isp_; + std::unique_ptr params_; + std::unique_ptr stats_; + + std::vector> statsBuffers_; + std::queue availableStatsBuffers_; + + std::vector> paramsBuffers_; + std::queue availableParamsBuffers_; + + std::map frameInfoMap_; + + std::array pipes_; +}; + +PipelineHandlerC3ISP::PipelineHandlerC3ISP(CameraManager *manager) + : PipelineHandler(manager) +{ +} + +std::unique_ptr +PipelineHandlerC3ISP::generateConfiguration(Camera *camera, + Span roles) +{ + C3ISPCameraData *data = cameraData(camera); + std::unique_ptr config = + std::make_unique(data); + bool isRoleRaw = false; + + if (roles.empty()) + return config; + + /* Reserve the C3ISPSTILL pipe for Raw role */ + if (std::find(roles.begin(), roles.end(), StreamRole::Raw) != roles.end()) + isRoleRaw = true; + + for (const StreamRole &role : roles) { + struct C3ISPPipe *pipe; + PixelFormat pixelFormat; + Size size = std::min(Size{ 1920, 1080 }, data->sensor_->resolution()); + + switch (role) { + case StreamRole::StillCapture: + size = data->sensor_->resolution(); + pixelFormat = formats::NV12; + pipe = isRoleRaw ? nullptr : &pipes_[C3ISPSTILL]; + break; + + case StreamRole::VideoRecording: + pixelFormat = formats::NV12; + pipe = &pipes_[C3ISPVIDEO]; + break; + + case StreamRole::Viewfinder: + pixelFormat = formats::NV12; + pipe = &pipes_[C3ISPVIEW]; + break; + + case StreamRole::Raw: + pixelFormat = data->bestRawFmt(); + if (!pixelFormat.isValid()) { + LOG(C3ISP, Error) + << "Failed to get Raw format"; + return nullptr; + } + + size = data->sensor_->resolution(); + pipe = &pipes_[C3ISPSTILL]; + break; + + default: + LOG(C3ISP, Error) << "Invalid stream role: " << role; + return nullptr; + } + + std::map> formats; + for (const auto &c3Format : C3ISPFmtToCode) { + PixelFormat pixFmt = c3Format.first; + bool isRaw = isFmtRaw(pixFmt); + + if (pipe != &pipes_[C3ISPSTILL] && isRaw) + continue; + + if (isRaw) { + int rawCode = data->pixfmtToMbusCode(pixFmt); + if (rawCode < 0) + continue; + + const auto sizes = data->sensor_->sizes(rawCode); + if (sizes.empty()) + continue; + + std::vector sizeRanges; + std::transform(sizes.begin(), sizes.end(), + std::back_inserter(sizeRanges), + [](const Size &s) { + return SizeRange(s); + }); + + formats[pixFmt] = sizeRanges; + } else { + Size maxSize = std::min(kC3ISPMaxSize, + data->sensor_->resolution()); + formats[pixFmt] = { kC3ISPMinSize, maxSize }; + } + } + + StreamFormats streamFormats(formats); + StreamConfiguration cfg(streamFormats); + cfg.pixelFormat = pixelFormat; + cfg.bufferCount = 4; + cfg.size = size; + + config->addConfiguration(cfg); + } + + if (config->validate() == CameraConfiguration::Invalid) + return nullptr; + + return config; +} + +int PipelineHandlerC3ISP::configureRawStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat) +{ + Stream *stream = config.stream(); + C3ISPPipe *pipe = pipeFromStream(data, stream); + + if (pipe != &pipes_[C3ISPSTILL]) { + LOG(C3ISP, Error) << "Failed to match the RAW pipe"; + return -EINVAL; + } + + const MediaEntity *resizerEntity = pipe->resizer->entity(); + int ret = resizerEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) { + LOG(C3ISP, Error) << "Couldn't enable resizer's sink link"; + return ret; + } + + auto it = C3ISPFmtToCode.find(config.pixelFormat); + if (it == C3ISPFmtToCode.end()) { + LOG(C3ISP, Error) << "Failed to find the RAW pixel format"; + return -EINVAL; + } + + subdevFormat.code = it->second; + ret = isp_->setFormat(5, &subdevFormat); + if (ret) + return ret; + + ret = pipe->resizer->setFormat(0, &subdevFormat); + if (ret) + return ret; + + return 0; +} + +int PipelineHandlerC3ISP::configureProcessedStream(C3ISPCameraData *data, + const StreamConfiguration &config, + V4L2SubdeviceFormat &subdevFormat) +{ + Stream *stream = config.stream(); + C3ISPPipe *pipe = pipeFromStream(data, stream); + + const MediaEntity *resizerEntity = pipe->resizer->entity(); + int ret = resizerEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + auto it = C3ISPFmtToCode.find(config.pixelFormat); + if (it == C3ISPFmtToCode.end()) { + LOG(C3ISP, Error) << "Failed to find the processed pixel format"; + return -EINVAL; + } + + unsigned int pad; + if (pipe == &pipes_[C3ISPVIEW]) + pad = 3; + else if (pipe == &pipes_[C3ISPVIDEO]) + pad = 4; + else if (pipe == &pipes_[C3ISPSTILL]) + pad = 5; + else { + LOG(C3ISP, Error) << "Failed to match the pipe"; + return -EINVAL; + } + + subdevFormat.code = it->second; + ret = isp_->setFormat(pad, &subdevFormat); + if (ret) + return ret; + + ret = pipe->resizer->setFormat(0, &subdevFormat); + if (ret) + return ret; + + /* Compute the scaler-in to scaler-out ratio: first center-crop to align + * the FOV to the desired resolution, then scale to the desired size. + */ + Size scalerIn = subdevFormat.size.boundedToAspectRatio(config.size); + int xCrop = (subdevFormat.size.width - scalerIn.width) / 2; + int yCrop = (subdevFormat.size.height - scalerIn.height) / 2; + + Rectangle cropRect = { xCrop, yCrop, scalerIn }; + ret = pipe->resizer->setSelection(0, V4L2_SEL_TGT_CROP, &cropRect); + if (ret) + return ret; + + Rectangle scaleRect = { 0, 0, config.size }; + ret = pipe->resizer->setSelection(0, V4L2_SEL_TGT_COMPOSE, &scaleRect); + if (ret) + return ret; + + subdevFormat.size = scaleRect.size(); + + return pipe->resizer->setFormat(1, &subdevFormat); +} + +int PipelineHandlerC3ISP::configure(Camera *camera, CameraConfiguration *config) +{ + resetPipes(); + + int ret = media_->disableLinks(); + if (ret) + return ret; + + C3ISPCameraData *data = cameraData(camera); + + /* Enable the link between sensor and CSI-2 */ + const MediaEntity *csiEntity = data->csi_->entity(); + ret = csiEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + /* Enable the link between CSI-2 and adapter */ + const MediaEntity *adapEntity = data->adap_->entity(); + ret = adapEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + /* Enable the link between adapter and ISP */ + const MediaEntity *ispEntity = isp_->entity(); + ret = ispEntity->getPadByIndex(0)->links()[0]->setEnabled(true); + if (ret) + return ret; + + C3ISPCameraConfiguration *c3Config = static_cast(config); + V4L2SubdeviceFormat subdevFormat = c3Config->sensorFormat_; + + ret = data->sensor_->setFormat(&subdevFormat, c3Config->combinedTransform()); + if (ret) + return ret; + + ret = data->csi_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + ret = data->adap_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + ret = isp_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + for (const StreamConfiguration &streamConfig : *config) { + Stream *stream = streamConfig.stream(); + C3ISPPipe *pipe = pipeFromStream(data, stream); + + if (isFmtRaw(streamConfig.pixelFormat)) + ret = configureRawStream(data, streamConfig, subdevFormat); + else + ret = configureProcessedStream(data, streamConfig, subdevFormat); + if (ret) { + LOG(C3ISP, Error) << "Failed to configure pipeline"; + return ret; + } + + V4L2DeviceFormat captureFormat; + captureFormat.fourcc = pipe->cap->toV4L2PixelFormat(streamConfig.pixelFormat); + captureFormat.size = streamConfig.size; + + ret = pipe->cap->setFormat(&captureFormat); + if (ret) + return ret; + + pipe->stream = stream; + } + + /* Enable the link between parameter node and ISP */ + ret = ispEntity->getPadByIndex(1)->links()[0]->setEnabled(true); + if (ret) { + LOG(C3ISP, Error) << "Failed to enable param link"; + return ret; + } + + /* Enable the link between ISP and 3A stats node */ + ret = ispEntity->getPadByIndex(2)->links()[0]->setEnabled(true); + if (ret) { + LOG(C3ISP, Error) << "Failed to enable stats link"; + return ret; + } + + /* Inform the IPA of the sensor configuration */ + ipa::c3isp::IPAConfigInfo ipaConfig{}; + + ret = data->sensor_->sensorInfo(&ipaConfig.sensorInfo); + if (ret) + return ret; + + ipaConfig.sensorControls = data->sensor_->controls(); + + ControlInfoMap ipaControls; + ret = data->ipa_->configure(ipaConfig, &ipaControls); + if (ret) { + LOG(C3ISP, Error) << "Failed to configure IPA"; + return ret; + } + + data->updateControls(ipaControls); + + return 0; +} + +int PipelineHandlerC3ISP::exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) +{ + C3ISPPipe *pipe = pipeFromStream(cameraData(camera), stream); + unsigned int count = stream->configuration().bufferCount; + + return pipe->cap->exportBuffers(count, buffers); +} + +int PipelineHandlerC3ISP::allocateBuffers(Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + unsigned int ipaBufferId = 1; + unsigned int bufferCount; + int ret; + + bufferCount = std::max({ data->viewStream.configuration().bufferCount, + data->videoStream.configuration().bufferCount, + data->stillStream.configuration().bufferCount }); + + ret = stats_->allocateBuffers(bufferCount, &statsBuffers_); + if (ret < 0) + return ret; + + for (std::unique_ptr &buffer : statsBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaStatBuffers_.emplace_back(buffer->cookie(), + buffer->planes()); + availableStatsBuffers_.push(buffer.get()); + } + + ret = params_->allocateBuffers(bufferCount, ¶msBuffers_); + if (ret < 0) + return ret; + + for (std::unique_ptr &buffer : paramsBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaParamBuffers_.emplace_back(buffer->cookie(), + buffer->planes()); + availableParamsBuffers_.push(buffer.get()); + } + + data->ipa_->mapBuffers(data->ipaStatBuffers_, true); + data->ipa_->mapBuffers(data->ipaParamBuffers_, false); + + return 0; +} + +void PipelineHandlerC3ISP::freeBuffers(Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + + while (!availableStatsBuffers_.empty()) + availableStatsBuffers_.pop(); + while (!availableParamsBuffers_.empty()) + availableParamsBuffers_.pop(); + + statsBuffers_.clear(); + paramsBuffers_.clear(); + + data->ipa_->unmapBuffers(data->ipaStatBuffers_); + data->ipa_->unmapBuffers(data->ipaParamBuffers_); + + data->ipaStatBuffers_.clear(); + data->ipaParamBuffers_.clear(); + + if (stats_->releaseBuffers()) + LOG(C3ISP, Error) << "Failed to release stats buffers"; + + if (params_->releaseBuffers()) + LOG(C3ISP, Error) << "Failed to release params buffers"; +} + +int PipelineHandlerC3ISP::pipesStart() +{ + for (C3ISPPipe &pipe : pipes_) { + if (!pipe.stream) + continue; + + Stream *stream = pipe.stream; + + int ret = pipe.cap->importBuffers(stream->configuration().bufferCount); + if (ret) { + LOG(C3ISP, Error) << "Failed to import buffers"; + return ret; + } + + ret = pipe.cap->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start stream"; + return ret; + } + } + + return 0; +} + +void PipelineHandlerC3ISP::pipesStop() +{ + for (C3ISPPipe &pipe : pipes_) { + if (!pipe.stream) + continue; + + pipe.cap->streamOff(); + pipe.cap->releaseBuffers(); + } +} + +int PipelineHandlerC3ISP::start(Camera *camera, + [[maybe_unused]] const ControlList *controls) +{ + C3ISPCameraData *data = cameraData(camera); + int ret; + + ret = allocateBuffers(camera); + if (ret) + return ret; + + ret = data->ipa_->start(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start IPA"; + + freeBuffers(camera); + return ret; + } + + ret = pipesStart(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start pipe"; + + data->ipa_->stop(); + freeBuffers(camera); + return ret; + } + + ret = stats_->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start stats"; + + pipesStop(); + data->ipa_->stop(); + freeBuffers(camera); + return ret; + } + + ret = params_->streamOn(); + if (ret) { + LOG(C3ISP, Error) << "Failed to start params"; + + stats_->streamOff(); + pipesStop(); + data->ipa_->stop(); + freeBuffers(camera); + return ret; + } + + ret = isp_->setFrameStartEnabled(true); + if (ret) { + LOG(C3ISP, Error) << "Failed to enable frame start"; + + params_->streamOff(); + stats_->streamOff(); + pipesStop(); + data->ipa_->stop(); + freeBuffers(camera); + return ret; + } + + return 0; +} + +void PipelineHandlerC3ISP::stopDevice(Camera *camera) +{ + C3ISPCameraData *data = cameraData(camera); + + isp_->setFrameStartEnabled(false); + + params_->streamOff(); + stats_->streamOff(); + data->ipa_->stop(); + freeBuffers(camera); + + pipesStop(); + clearIncompleteRequests(); +} + +int PipelineHandlerC3ISP::queueRequestDevice(Camera *camera, Request *request) +{ + C3ISPCameraData *data = cameraData(camera); + + if (availableStatsBuffers_.empty()) { + LOG(C3ISP, Error) << "No available stats buffer"; + return -ENOENT; + } + + if (availableParamsBuffers_.empty()) { + LOG(C3ISP, Error) << "No available params buffer"; + return -ENOENT; + } + + C3ISPFrameInfo frameInfo; + frameInfo.request = request; + + frameInfo.statBuffer = availableStatsBuffers_.front(); + availableStatsBuffers_.pop(); + frameInfo.paramBuffer = availableParamsBuffers_.front(); + availableParamsBuffers_.pop(); + + frameInfo.paramsDone = false; + frameInfo.statsDone = false; + + frameInfoMap_[request->sequence()] = frameInfo; + + data->ipa_->queueRequest(request->sequence(), request->controls()); + data->ipa_->computeParams(request->sequence(), + frameInfo.paramBuffer->cookie()); + + return 0; +} + +C3ISPFrameInfo *PipelineHandlerC3ISP::findFrameInfo(Request *request) +{ + for (auto &[sequence, info] : frameInfoMap_) { + if (info.request == request) + return &info; + } + + return nullptr; +} + +C3ISPFrameInfo *PipelineHandlerC3ISP::findFrameInfo(FrameBuffer *buffer) +{ + for (auto &[sequence, info] : frameInfoMap_) { + if (info.paramBuffer == buffer || info.statBuffer == buffer) + return &info; + } + + return nullptr; +} + +void PipelineHandlerC3ISP::clearIncompleteRequests() +{ + for (auto &[sequence, info] : frameInfoMap_) { + if (info.request) + cancelRequest(info.request); + } + + frameInfoMap_.clear(); +} + +void PipelineHandlerC3ISP::tryComplete(C3ISPFrameInfo *info) +{ + if (!info->paramsDone) + return; + + if (!info->statsDone) + return; + + Request *request = info->request; + if (request->hasPendingBuffers()) + return; + + if (info->statBuffer) + availableStatsBuffers_.push(info->statBuffer); + + if (info->paramBuffer) + availableParamsBuffers_.push(info->paramBuffer); + + frameInfoMap_.erase(request->sequence()); + + completeRequest(request); +} + +void PipelineHandlerC3ISP::imageBufferReady(FrameBuffer *buffer) +{ + Request *request = buffer->request(); + C3ISPFrameInfo *frameInfo = findFrameInfo(request); + ASSERT(frameInfo); + + if (completeBuffer(request, buffer)) + tryComplete(frameInfo); +} + +void PipelineHandlerC3ISP::paramsBufferReady(FrameBuffer *buffer) +{ + C3ISPFrameInfo *frameInfo = findFrameInfo(buffer); + ASSERT(frameInfo); + + frameInfo->paramsDone = true; + + tryComplete(frameInfo); +} + +void PipelineHandlerC3ISP::statsBufferReady(FrameBuffer *buffer) +{ + C3ISPFrameInfo *frameInfo = findFrameInfo(buffer); + ASSERT(frameInfo); + + Request *request = frameInfo->request; + C3ISPCameraData *data = cameraData(request->_d()->camera()); + + ControlList sensorControls = data->delayedCtrls_->get(buffer->metadata().sequence); + + data->ipa_->processStats(request->sequence(), buffer->cookie(), + sensorControls); +} + +void PipelineHandlerC3ISP::paramsComputed(unsigned int requestId, + unsigned int bytesused) +{ + C3ISPFrameInfo &frameInfo = frameInfoMap_[requestId]; + Request *request = frameInfo.request; + C3ISPCameraData *data = cameraData(request->_d()->camera()); + + frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused; + + params_->queueBuffer(frameInfo.paramBuffer); + stats_->queueBuffer(frameInfo.statBuffer); + + for (auto &[stream, buffer] : request->buffers()) { + C3ISPPipe *pipe = pipeFromStream(data, stream); + + pipe->cap->queueBuffer(buffer); + } +} + +void PipelineHandlerC3ISP::statsProcessed(unsigned int requestId, + const ControlList &metadata) +{ + C3ISPFrameInfo &frameInfo = frameInfoMap_[requestId]; + + frameInfo.statsDone = true; + frameInfo.request->metadata().merge(metadata); + + tryComplete(&frameInfo); +} + +bool PipelineHandlerC3ISP::createCamera(MediaLink *ispLink) +{ + MediaEntity *adap = ispLink->source()->entity(); + const MediaPad *adapSink = adap->getPadByIndex(0); + MediaEntity *csi = adapSink->links()[0]->source()->entity(); + const MediaPad *csiSink = csi->getPadByIndex(0); + + MediaEntity *sensor = csiSink->links()[0]->source()->entity(); + if (sensor->function() != MEDIA_ENT_F_CAM_SENSOR) + return false; + + std::unique_ptr data = + std::make_unique(this, sensor); + if (data->init()) + return false; + + /* Generic values for sensor */ + const CameraSensorProperties::SensorDelays &delays = data->sensor_->sensorDelays(); + std::unordered_map params = { + { V4L2_CID_ANALOGUE_GAIN, { delays.gainDelay, false } }, + { V4L2_CID_EXPOSURE, { delays.exposureDelay, false } }, + }; + + data->delayedCtrls_ = + std::make_unique(data->sensor_->device(), params); + isp_->frameStart.connect(data->delayedCtrls_.get(), + &DelayedControls::applyControls); + + if (data->loadIPA()) + return false; + + data->ipa_->statsProcessed.connect(this, &PipelineHandlerC3ISP::statsProcessed); + data->ipa_->paramsComputed.connect(this, &PipelineHandlerC3ISP::paramsComputed); + + std::set streams{ &data->viewStream, &data->stillStream, &data->videoStream }; + + std::shared_ptr camera = Camera::create(std::move(data), sensor->name(), streams); + + registerCamera(std::move(camera)); + + return true; +} + +bool PipelineHandlerC3ISP::match(DeviceEnumerator *enumerator) +{ + const MediaPad *ispSink; + + DeviceMatch dm("c3-isp"); + dm.add("c3-mipi-csi2"); + dm.add("c3-mipi-adapter"); + dm.add("c3-isp-core"); + + media_ = acquireMediaDevice(enumerator, dm); + if (!media_) + return false; + + isp_ = V4L2Subdevice::fromEntityName(media_, "c3-isp-core"); + if (isp_->open() < 0) + return false; + + stats_ = V4L2VideoDevice::fromEntityName(media_, "c3-isp-stats"); + if (stats_->open() < 0) + return false; + + params_ = V4L2VideoDevice::fromEntityName(media_, "c3-isp-params"); + if (params_->open() < 0) + return false; + + for (unsigned int i = C3ISPVIEW; i < C3ISPNumPipes; i++) { + C3ISPPipe *ispPipe = &pipes_[i]; + + std::ostringstream resizer; + resizer << "c3-isp-resizer" << i; + ispPipe->resizer = V4L2Subdevice::fromEntityName(media_, resizer.str()); + if (ispPipe->resizer->open() < 0) + return false; + + std::ostringstream capture; + capture << "c3-isp-cap" << i; + ispPipe->cap = V4L2VideoDevice::fromEntityName(media_, capture.str()); + if (ispPipe->cap->open() < 0) + return false; + + ispPipe->cap->bufferReady.connect(this, &PipelineHandlerC3ISP::imageBufferReady); + } + + stats_->bufferReady.connect(this, &PipelineHandlerC3ISP::statsBufferReady); + params_->bufferReady.connect(this, &PipelineHandlerC3ISP::paramsBufferReady); + + ispSink = isp_->entity()->getPadByIndex(0); + if (!ispSink || ispSink->links().empty()) + return false; + + if (!createCamera(ispSink->links()[0])) { + LOG(C3ISP, Error) << "Failed to create camera"; + return false; + } + + return true; +} + +REGISTER_PIPELINE_HANDLER(PipelineHandlerC3ISP, "c3isp") + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/c3-isp/meson.build b/src/libcamera/pipeline/c3-isp/meson.build new file mode 100644 index 00000000..5f8b23f1 --- /dev/null +++ b/src/libcamera/pipeline/c3-isp/meson.build @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: CC0-1.0 + +libcamera_internal_sources += files([ + 'c3-isp.cpp' +])