From patchwork Sat Feb 13 04:22:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 11286 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D7858BD160 for ; Sat, 13 Feb 2021 04:23:09 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 9E6E1637C3; Sat, 13 Feb 2021 05:23:09 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="srAod0Tj"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4FD606379C for ; Sat, 13 Feb 2021 05:23:08 +0100 (CET) Received: from pyrite.rasen.tech (unknown [IPv6:2400:4051:61:600:2c71:1b79:d06d:5032]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 3C61D1576; Sat, 13 Feb 2021 05:23:05 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1613190188; bh=4Zp9YBk/LWPGHbY/jo8Cw9vpZ43YBouSIhPsbhzQ/Ug=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=srAod0TjNIULgYXlSIUOOwowbYu9uzGjCxFzmqLKwCrsYf67Fm2gIAdwCYNx0Fk+R QwOF+8uTDKCIv+Wd3R3w7ki68VFY/028GTpYhT+mcpdL3hYZnpZ2yLnkguednVh3cU TtF1OZuLV7MjKeFfBOQu0jcKZiFNLRA0qJ9cWGk8= From: Paul Elder To: libcamera-devel@lists.libcamera.org Date: Sat, 13 Feb 2021 13:22:25 +0900 Message-Id: <20210213042225.112477-13-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210213042225.112477-1-paul.elder@ideasonboard.com> References: <20210213042225.112477-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v8 12/12] libcamera: pipeline, ipa: ipu3: Support the new IPC mechanism X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Niklas Söderlund Add support to ipu3 pipeline handler and IPA for the new IPC mechanism. [Original version] Signed-off-by: Niklas Söderlund Reviewed-by: Paul Elder [Fixed commit message and small changes] Signed-off-by: Paul Elder Reviewed-by: Laurent Pinchart --- Changes in v8: - replace push_back with emplace_back New in v7 - was "ipu3: Translate IPA protocol to new mojo interface" - remove include core_ipa_interface.h from pipeline ipu3.cpp - change commit message - move removal of include ipa_interface_wrapper.h to "libcamera: IPAInterface: Replace C API with the new C++-only API" --- include/libcamera/ipa/ipu3.h | 23 ------- include/libcamera/ipa/ipu3.mojom | 43 +++++++++++++ include/libcamera/ipa/meson.build | 1 + src/ipa/ipu3/ipu3.cpp | 72 +++++++++------------ src/ipa/ipu3/meson.build | 4 +- src/libcamera/pipeline/ipu3/ipu3.cpp | 93 +++++++++++----------------- 6 files changed, 109 insertions(+), 127 deletions(-) delete mode 100644 include/libcamera/ipa/ipu3.h create mode 100644 include/libcamera/ipa/ipu3.mojom diff --git a/include/libcamera/ipa/ipu3.h b/include/libcamera/ipa/ipu3.h deleted file mode 100644 index cbaaef04..00000000 --- a/include/libcamera/ipa/ipu3.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1-or-later */ -/* - * Copyright (C) 2020, Google Inc. - * - * ipu3.h - Image Processing Algorithm interface for IPU3 - */ -#ifndef __LIBCAMERA_IPA_INTERFACE_IPU3_H__ -#define __LIBCAMERA_IPA_INTERFACE_IPU3_H__ - -#ifndef __DOXYGEN__ - -enum IPU3Operations { - IPU3_IPA_ACTION_SET_SENSOR_CONTROLS = 1, - IPU3_IPA_ACTION_PARAM_FILLED = 2, - IPU3_IPA_ACTION_METADATA_READY = 3, - IPU3_IPA_EVENT_PROCESS_CONTROLS = 4, - IPU3_IPA_EVENT_STAT_READY = 5, - IPU3_IPA_EVENT_FILL_PARAMS = 6, -}; - -#endif /* __DOXYGEN__ */ - -#endif /* __LIBCAMERA_IPA_INTERFACE_IPU3_H__ */ diff --git a/include/libcamera/ipa/ipu3.mojom b/include/libcamera/ipa/ipu3.mojom new file mode 100644 index 00000000..6ee11333 --- /dev/null +++ b/include/libcamera/ipa/ipu3.mojom @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +module ipa.ipu3; + +import "include/libcamera/ipa/core.mojom"; + +enum IPU3Operations { + ActionSetSensorControls = 1, + ActionParamFilled = 2, + ActionMetadataReady = 3, + EventProcessControls = 4, + EventStatReady = 5, + EventFillParams = 6, +}; + +struct IPU3Event { + IPU3Operations op; + uint32 frame; + uint32 bufferId; + ControlList controls; +}; + +struct IPU3Action { + IPU3Operations op; + ControlList controls; +}; + +interface IPAIPU3Interface { + init(IPASettings settings) => (int32 ret); + start() => (int32 ret); + stop(); + + configure(map entityControls) => (); + + mapBuffers(array buffers); + unmapBuffers(array ids); + + [async] processEvent(IPU3Event ev); +}; + +interface IPAIPU3EventInterface { + queueFrameAction(uint32 frame, IPU3Action action); +}; diff --git a/include/libcamera/ipa/meson.build b/include/libcamera/ipa/meson.build index d701bccc..fe8aa65b 100644 --- a/include/libcamera/ipa/meson.build +++ b/include/libcamera/ipa/meson.build @@ -55,6 +55,7 @@ libcamera_generated_ipa_headers += custom_target('core_ipa_serializer_h', ]) ipa_mojom_files = [ + 'ipu3.mojom', 'raspberrypi.mojom', 'rkisp1.mojom', 'vimc.mojom', diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index d5bde223..fcd8889c 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -5,8 +5,6 @@ * ipu3.cpp - IPU3 Image Processing Algorithms */ -#include - #include #include @@ -17,6 +15,7 @@ #include #include #include +#include #include #include "libcamera/internal/buffer.h" @@ -26,25 +25,21 @@ namespace libcamera { LOG_DEFINE_CATEGORY(IPAIPU3) -class IPAIPU3 : public IPAInterface +class IPAIPU3 : public ipa::ipu3::IPAIPU3Interface { public: int init([[maybe_unused]] const IPASettings &settings) override { return 0; } - int start([[maybe_unused]] const IPAOperationData &data, - [[maybe_unused]] IPAOperationData *result) override { return 0; } + int start() override { return 0; } void stop() override {} - void configure(const CameraSensorInfo &info, - const std::map &streamConfig, - const std::map &entityControls, - const IPAOperationData &ipaConfig, - IPAOperationData *response) override; + void configure(const std::map &entityControls) override; + void mapBuffers(const std::vector &buffers) override; void unmapBuffers(const std::vector &ids) override; - void processEvent(const IPAOperationData &event) override; + void processEvent(const ipa::ipu3::IPU3Event &event) override; private: void processControls(unsigned int frame, const ControlList &controls); @@ -67,11 +62,7 @@ private: uint32_t maxGain_; }; -void IPAIPU3::configure([[maybe_unused]] const CameraSensorInfo &info, - [[maybe_unused]] const std::map &streamConfig, - const std::map &entityControls, - [[maybe_unused]] const IPAOperationData &ipaConfig, - [[maybe_unused]] IPAOperationData *result) +void IPAIPU3::configure(const std::map &entityControls) { if (entityControls.empty()) return; @@ -121,19 +112,15 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) } } -void IPAIPU3::processEvent(const IPAOperationData &event) +void IPAIPU3::processEvent(const ipa::ipu3::IPU3Event &event) { - switch (event.operation) { - case IPU3_IPA_EVENT_PROCESS_CONTROLS: { - unsigned int frame = event.data[0]; - processControls(frame, event.controls[0]); + switch (event.op) { + case ipa::ipu3::EventProcessControls: { + processControls(event.frame, event.controls); break; } - case IPU3_IPA_EVENT_STAT_READY: { - unsigned int frame = event.data[0]; - unsigned int bufferId = event.data[1]; - - auto it = buffers_.find(bufferId); + case ipa::ipu3::EventStatReady: { + auto it = buffers_.find(event.bufferId); if (it == buffers_.end()) { LOG(IPAIPU3, Error) << "Could not find stats buffer!"; return; @@ -143,14 +130,11 @@ void IPAIPU3::processEvent(const IPAOperationData &event) const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); - parseStatistics(frame, stats); + parseStatistics(event.frame, stats); break; } - case IPU3_IPA_EVENT_FILL_PARAMS: { - unsigned int frame = event.data[0]; - unsigned int bufferId = event.data[1]; - - auto it = buffers_.find(bufferId); + case ipa::ipu3::EventFillParams: { + auto it = buffers_.find(event.bufferId); if (it == buffers_.end()) { LOG(IPAIPU3, Error) << "Could not find param buffer!"; return; @@ -160,11 +144,11 @@ void IPAIPU3::processEvent(const IPAOperationData &event) ipu3_uapi_params *params = reinterpret_cast(mem.data()); - fillParams(frame, params); + fillParams(event.frame, params); break; } default: - LOG(IPAIPU3, Error) << "Unknown event " << event.operation; + LOG(IPAIPU3, Error) << "Unknown event " << event.op; break; } } @@ -182,8 +166,8 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) /* \todo Fill in parameters buffer. */ - IPAOperationData op; - op.operation = IPU3_IPA_ACTION_PARAM_FILLED; + ipa::ipu3::IPU3Action op; + op.op = ipa::ipu3::ActionParamFilled; queueFrameAction.emit(frame, op); @@ -199,22 +183,22 @@ void IPAIPU3::parseStatistics(unsigned int frame, /* \todo React to statistics and update internal state machine. */ /* \todo Add meta-data information to ctrls. */ - IPAOperationData op; - op.operation = IPU3_IPA_ACTION_METADATA_READY; - op.controls.push_back(ctrls); + ipa::ipu3::IPU3Action op; + op.op = ipa::ipu3::ActionMetadataReady; + op.controls = ctrls; queueFrameAction.emit(frame, op); } void IPAIPU3::setControls(unsigned int frame) { - IPAOperationData op; - op.operation = IPU3_IPA_ACTION_SET_SENSOR_CONTROLS; + ipa::ipu3::IPU3Action op; + op.op = ipa::ipu3::ActionSetSensorControls; ControlList ctrls(ctrls_); ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_)); - op.controls.push_back(ctrls); + op.controls = ctrls; queueFrameAction.emit(frame, op); } @@ -231,9 +215,9 @@ const struct IPAModuleInfo ipaModuleInfo = { "ipu3", }; -struct ipa_context *ipaCreate() +IPAInterface *ipaCreate() { - return new IPAInterfaceWrapper(std::make_unique()); + return new IPAIPU3(); } } diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index d31594fc..a241f617 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -3,10 +3,10 @@ ipa_name = 'ipa_ipu3' mod = shared_module(ipa_name, - 'ipu3.cpp', + ['ipu3.cpp', libcamera_generated_ipa_headers], name_prefix : '', include_directories : [ipa_includes, libipa_includes], - dependencies : [libatomic, libcamera_dep], + dependencies : libcamera_dep, link_with : libipa, install : true, install_dir : ipa_install_dir) diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp index 61f7bf43..3e6b88af 100644 --- a/src/libcamera/pipeline/ipu3/ipu3.cpp +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp @@ -14,7 +14,8 @@ #include #include #include -#include +#include +#include #include #include @@ -77,8 +78,11 @@ public: std::unique_ptr delayedCtrls_; IPU3Frames frameInfos_; + std::unique_ptr ipa_; + private: - void queueFrameAction(unsigned int id, const IPAOperationData &op); + void queueFrameAction(unsigned int id, + const ipa::ipu3::IPU3Action &action); }; class IPU3CameraConfiguration : public CameraConfiguration @@ -609,18 +613,12 @@ int PipelineHandlerIPU3::allocateBuffers(Camera *camera) for (const std::unique_ptr &buffer : imgu->paramBuffers_) { buffer->setCookie(ipaBufferId++); - ipaBuffers_.push_back({ - .id = buffer->cookie(), - .planes = buffer->planes() - }); + ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); } for (const std::unique_ptr &buffer : imgu->statBuffers_) { buffer->setCookie(ipaBufferId++); - ipaBuffers_.push_back({ - .id = buffer->cookie(), - .planes = buffer->planes() - }); + ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); } data->ipa_->mapBuffers(ipaBuffers_); @@ -650,16 +648,10 @@ int PipelineHandlerIPU3::freeBuffers(Camera *camera) int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] ControlList *controls) { + std::map entityControls; IPU3CameraData *data = cameraData(camera); CIO2Device *cio2 = &data->cio2_; ImgUDevice *imgu = data->imgu_; - - CameraSensorInfo sensorInfo = {}; - std::map streamConfig; - std::map entityControls; - IPAOperationData ipaConfig; - IPAOperationData result = {}; - int ret; /* Allocate buffers for internal pipeline usage. */ @@ -667,8 +659,7 @@ int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] ControlList *con if (ret) return ret; - IPAOperationData ipaData = {}; - ret = data->ipa_->start(ipaData, nullptr); + ret = data->ipa_->start(); if (ret) goto error; @@ -684,24 +675,8 @@ int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] ControlList *con if (ret) goto error; - /* Inform IPA of stream configuration and sensor controls. */ - ret = data->cio2_.sensor()->sensorInfo(&sensorInfo); - if (ret) - goto error; - - streamConfig[0] = { - .pixelFormat = data->outStream_.configuration().pixelFormat, - .size = data->outStream_.configuration().size, - }; - streamConfig[1] = { - .pixelFormat = data->vfStream_.configuration().pixelFormat, - .size = data->vfStream_.configuration().size, - }; - entityControls.emplace(0, data->cio2_.sensor()->controls()); - - data->ipa_->configure(sensorInfo, streamConfig, entityControls, - ipaConfig, &result); + data->ipa_->configure(entityControls); return 0; @@ -751,11 +726,11 @@ int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request) info->rawBuffer = rawBuffer; - IPAOperationData op; - op.operation = IPU3_IPA_EVENT_PROCESS_CONTROLS; - op.data = { info->id }; - op.controls = { request->controls() }; - data->ipa_->processEvent(op); + ipa::ipu3::IPU3Event ev; + ev.op = ipa::ipu3::EventProcessControls; + ev.frame = info->id; + ev.controls = request->controls(); + data->ipa_->processEvent(ev); return 0; } @@ -1048,7 +1023,7 @@ int PipelineHandlerIPU3::registerCameras() int IPU3CameraData::loadIPA() { - ipa_ = IPAManager::createIPA(pipe_, 1, 1); + ipa_ = IPAManager::createIPA(pipe_, 1, 1); if (!ipa_) return -ENOENT; @@ -1060,15 +1035,15 @@ int IPU3CameraData::loadIPA() } void IPU3CameraData::queueFrameAction(unsigned int id, - const IPAOperationData &action) + const ipa::ipu3::IPU3Action &action) { - switch (action.operation) { - case IPU3_IPA_ACTION_SET_SENSOR_CONTROLS: { - const ControlList &controls = action.controls[0]; + switch (action.op) { + case ipa::ipu3::ActionSetSensorControls: { + const ControlList &controls = action.controls; delayedCtrls_->push(controls); break; } - case IPU3_IPA_ACTION_PARAM_FILLED: { + case ipa::ipu3::ActionParamFilled: { IPU3Frames::Info *info = frameInfos_.find(id); if (!info) break; @@ -1090,13 +1065,13 @@ void IPU3CameraData::queueFrameAction(unsigned int id, break; } - case IPU3_IPA_ACTION_METADATA_READY: { + case ipa::ipu3::ActionMetadataReady: { IPU3Frames::Info *info = frameInfos_.find(id); if (!info) break; Request *request = info->request; - request->metadata() = action.controls[0]; + request->metadata() = action.controls; info->metadataProcessed = true; if (frameInfos_.tryComplete(info)) pipe_->completeRequest(request); @@ -1104,7 +1079,7 @@ void IPU3CameraData::queueFrameAction(unsigned int id, break; } default: - LOG(IPU3, Error) << "Unknown action " << action.operation; + LOG(IPU3, Error) << "Unknown action " << action.op; break; } } @@ -1172,10 +1147,11 @@ void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer) if (request->findBuffer(&rawStream_)) pipe_->completeBuffer(request, buffer); - IPAOperationData op; - op.operation = IPU3_IPA_EVENT_FILL_PARAMS; - op.data = { info->id, info->paramBuffer->cookie() }; - ipa_->processEvent(op); + ipa::ipu3::IPU3Event ev; + ev.op = ipa::ipu3::EventFillParams; + ev.frame = info->id; + ev.bufferId = info->paramBuffer->cookie(); + ipa_->processEvent(ev); } void IPU3CameraData::paramBufferReady(FrameBuffer *buffer) @@ -1202,10 +1178,11 @@ void IPU3CameraData::statBufferReady(FrameBuffer *buffer) return; } - IPAOperationData op; - op.operation = IPU3_IPA_EVENT_STAT_READY; - op.data = { info->id, info->statBuffer->cookie() }; - ipa_->processEvent(op); + ipa::ipu3::IPU3Event ev; + ev.op = ipa::ipu3::EventStatReady; + ev.frame = info->id; + ev.bufferId = info->statBuffer->cookie(); + ipa_->processEvent(ev); } REGISTER_PIPELINE_HANDLER(PipelineHandlerIPU3)