From patchwork Thu Jul 3 11:42:19 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 23734 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 80A5EC3293 for ; Thu, 3 Jul 2025 11:42:54 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id DDF3468E59; Thu, 3 Jul 2025 13:42:53 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="ryKJzJ1S"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1BBF068E41 for ; Thu, 3 Jul 2025 13:42:52 +0200 (CEST) Received: from neptunite.hamster-moth.ts.net (unknown [IPv6:2404:7a81:160:2100:c61b:f3bf:2578:6674]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B5C7E593; Thu, 3 Jul 2025 13:42:26 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1751542948; bh=LB4Fi+5TVnj5LfL7dTH97XCzPLl0UKvl9mZCQw88JsE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ryKJzJ1SolvyvF1bsHTU0YyGXIRJNsfjOQE9bltMztoQsWbmytqhhNWAUfOI4jnWM nhlZb3zTNMVjnaQtg+4jfqS0nvCJCXaJWYFgpj61x2SK+qfOKp+gywROMKe6GmJLX/ Tj/eAkkNse9ZnfrFEoVcz8nqkpzaPRBjGIYWBNJA= From: Paul Elder To: libcamera-devel@lists.libcamera.org Cc: Paul Elder , kieran.bingham@ideasonboard.com, barnabas.pocze@ideasonboard.com Subject: [PATCH v2 4/8] libcamera: layer_manager: Add LayerManager implementation Date: Thu, 3 Jul 2025 20:42:19 +0900 Message-ID: <20250703114225.2074071-5-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703114225.2074071-1-paul.elder@ideasonboard.com> References: <20250703114225.2074071-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" We want to be able to implement layers in libcamera, which conceptually sit in between the Camera class and the application. This can be useful for implementing things that don't belong inside the Camera/IPA nor inside the application, such as intercepting and translation the AeEnable control, or implementing the Sync algorithm. To achieve this, first add a LayerManager implementation, which searches for and loads layers from shared object files, and orchestrates executing them. Actually calling into these functions from the Camera class will be added in the following patch. Signed-off-by: Paul Elder --- Changes in v2: - add closures to the layer interface - separate Layer into LayerInfo and LayerInterface, to separate out layer data and the vtable - remove generateConfiguration() and streams() from the layer interface - add init() and terminate() to the layer interface, and make them required (mainly for preparing and destroying the closure) - make LayerLoaded automatically dlclose on deconstruction, remove copy, and implement move constructors - cache controls and properties in the LayerManager --- include/libcamera/internal/layer_manager.h | 117 +++++++ include/libcamera/internal/meson.build | 1 + include/libcamera/layer.h | 54 +++ include/libcamera/meson.build | 1 + src/layer/meson.build | 10 + src/libcamera/layer_manager.cpp | 383 +++++++++++++++++++++ src/libcamera/meson.build | 1 + src/meson.build | 1 + 8 files changed, 568 insertions(+) create mode 100644 include/libcamera/internal/layer_manager.h create mode 100644 include/libcamera/layer.h create mode 100644 src/layer/meson.build create mode 100644 src/libcamera/layer_manager.cpp diff --git a/include/libcamera/internal/layer_manager.h b/include/libcamera/internal/layer_manager.h new file mode 100644 index 000000000000..0d108bcddf3d --- /dev/null +++ b/include/libcamera/internal/layer_manager.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Ideas On Board Oy + * + * Layer manager interface + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(LayerManager) + +class LayerManager +{ +public: + LayerManager(); + ~LayerManager() = default; + + void init(const Camera *camera, const ControlList &properties, + const ControlInfoMap &controlInfoMap); + void terminate(const Camera *camera); + + void bufferCompleted(const Camera *camera, + Request *request, FrameBuffer *buffer); + void requestCompleted(const Camera *camera, Request *request); + void disconnected(const Camera *camera); + + void acquire(const Camera *camera); + void release(const Camera *camera); + + const ControlInfoMap &controls(const Camera *camera) const { return controls_.at(camera); } + const ControlList &properties(const Camera *camera) const { return properties_.at(camera); } + + void configure(const Camera *camera, const CameraConfiguration *config, + const ControlInfoMap &controlInfoMap); + + void createRequest(const Camera *camera, + uint64_t cookie, const Request *request); + + void queueRequest(const Camera *camera, Request *request); + + void start(const Camera *camera, const ControlList *controls); + void stop(const Camera *camera); + +private: + /* Extend the layer with information specific to load-handling */ + struct LayerLoaded + { + LayerLoaded() + : info(nullptr), vtable(nullptr), dlHandle(nullptr) + { + } + + LayerLoaded(LayerLoaded &&other) + : info(other.info), vtable(other.vtable), + dlHandle(other.dlHandle) + { + other.dlHandle = nullptr; + } + + LayerLoaded &operator=(LayerLoaded &&other) + { + info = other.info; + vtable = other.vtable; + dlHandle = other.dlHandle; + other.dlHandle = nullptr; + return *this; + } + + ~LayerLoaded() + { + if (dlHandle) + dlclose(dlHandle); + } + + LayerInfo *info; + LayerInterface *vtable; + void *dlHandle; + + private: + LIBCAMERA_DISABLE_COPY(LayerLoaded) + }; + + using ClosureKey = std::tuple; + + void updateProperties(const Camera *camera, + const ControlList &properties); + void updateControls(const Camera *camera, + const ControlInfoMap &controlInfoMap); + + LayerLoaded createLayer(const std::string &file); + std::deque executionQueue_; + std::map closures_; + + std::map controls_; + std::map properties_; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 690f5c5ec9f6..20e6c295601f 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -29,6 +29,7 @@ libcamera_internal_headers = files([ 'ipa_proxy.h', 'ipc_pipe.h', 'ipc_unixsocket.h', + 'layer_manager.h', 'mapped_framebuffer.h', 'matrix.h', 'media_device.h', diff --git a/include/libcamera/layer.h b/include/libcamera/layer.h new file mode 100644 index 000000000000..cd0e26a3b72b --- /dev/null +++ b/include/libcamera/layer.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Ideas On Board Oy + * + * Layer interface + */ + +#pragma once + +#include +#include + +#include + +#include + +namespace libcamera { + +class CameraConfiguration; +class FrameBuffer; +class Request; +class Stream; +enum class StreamRole; + +struct LayerInfo { + const char *name; + int layerAPIVersion; +}; + +struct LayerInterface { + void *(*init)(const std::string &id); + void (*terminate)(void *); + + void (*bufferCompleted)(void *, Request *, FrameBuffer *); + void (*requestCompleted)(void *, Request *); + void (*disconnected)(void *); + + void (*acquire)(void *); + void (*release)(void *); + + ControlInfoMap::Map (*controls)(void *, ControlInfoMap &); + ControlList (*properties)(void *, ControlList &); + + void (*configure)(void *, const CameraConfiguration *); + + void (*createRequest)(void *, uint64_t, const Request *); + + void (*queueRequest)(void *, Request *); + + void (*start)(void *, const ControlList *); + void (*stop)(void *); +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/meson.build b/include/libcamera/meson.build index 30ea76f9470a..552af112abb5 100644 --- a/include/libcamera/meson.build +++ b/include/libcamera/meson.build @@ -11,6 +11,7 @@ libcamera_public_headers = files([ 'framebuffer.h', 'framebuffer_allocator.h', 'geometry.h', + 'layer.h', 'logging.h', 'orientation.h', 'pixel_format.h', diff --git a/src/layer/meson.build b/src/layer/meson.build new file mode 100644 index 000000000000..dee5e5ac5804 --- /dev/null +++ b/src/layer/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +layer_includes = [ + libcamera_includes, +] + +layer_install_dir = libcamera_libdir / 'layers' + +config_h.set('LAYER_DIR', + '"' + get_option('prefix') / layer_install_dir + '"') diff --git a/src/libcamera/layer_manager.cpp b/src/libcamera/layer_manager.cpp new file mode 100644 index 000000000000..d707d4e12a53 --- /dev/null +++ b/src/libcamera/layer_manager.cpp @@ -0,0 +1,383 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Ideas On Board Oy + * + * Layer manager + */ + +#include "libcamera/internal/layer_manager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "libcamera/internal/utils.h" + +/** + * \file layer_manager.h + * \brief Layer manager + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(LayerManager) + +/** + * \class LayerManager + * \brief Layer manager + * + * The Layer manager discovers layer implementations from disk, and creates + * execution queues for every function that is implemented by each layer and + * executes them. A layer is a layer that sits between libcamera and the + * application, and hooks into the public Camera interface. + */ + +/** + * \brief Construct a LayerManager instance + * + * The LayerManager class is meant be instantiated by the Camera. + */ +LayerManager::LayerManager() +{ + std::map layers; + + /* \todo Implement built-in layers */ + + /* This returns the number of "modules" successfully loaded */ + std::function addDirHandler = + [this, &layers](const std::string &file) { + LayerManager::LayerLoaded layer = createLayer(file); + if (!layer.info) + return 0; + + LOG(LayerManager, Debug) << "Loaded layer '" << file << "'"; + + layers.emplace(std::string(layer.info->name), std::move(layer)); + + return 1; + }; + + /* User-specified paths take precedence. */ + /* \todo Document this */ + const char *layerPaths = utils::secure_getenv("LIBCAMERA_LAYER_PATH"); + if (layerPaths) { + for (const auto &dir : utils::split(layerPaths, ":")) { + if (dir.empty()) + continue; + + /* + * \todo Move the shared objects into one directory + * instead of each in their own subdir + */ + utils::addDir(dir.c_str(), 1, addDirHandler); + } + } + + /* + * When libcamera is used before it is installed, load layers from the + * same build directory as the libcamera library itself. + */ + std::string root = utils::libcameraBuildPath(); + if (!root.empty()) { + std::string layerBuildPath = root + "src/layer"; + constexpr int maxDepth = 2; + + LOG(LayerManager, Info) + << "libcamera is not installed. Adding '" + << layerBuildPath << "' to the layer search path"; + + utils::addDir(layerBuildPath.c_str(), maxDepth, addDirHandler); + } + + /* Finally try to load layers from the installed system path. */ + utils::addDir(LAYER_DIR, 1, addDirHandler); + + /* Order the layers */ + /* \todo Document this. First is closer to application, last is closer to libcamera */ + const char *layerList = utils::secure_getenv("LIBCAMERA_LAYERS_ENABLE"); + if (layerList) { + for (const auto &layerName : utils::split(layerList, ":")) { + if (layerName.empty()) + continue; + + const auto &it = layers.find(layerName); + if (it == layers.end()) + continue; + + executionQueue_.push_back(std::move(it->second)); + } + } +} + +void LayerManager::init(const Camera *camera, const ControlList &properties, + const ControlInfoMap &controlInfoMap) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + void *closure = layer.vtable->init(camera->id()); + closures_[std::make_tuple(camera, &layer)] = closure; + } + + /* + * We need to iterate over the layers individually to merge all of + * their controls, so we'll factor out updateControls() as it needs to be + * run again at configure(). + */ + updateProperties(camera, properties); + updateControls(camera, controlInfoMap); +} + +void LayerManager::terminate(const Camera *camera) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->terminate(closure); + } +} + +LayerManager::LayerLoaded LayerManager::createLayer(const std::string &filename) +{ + LayerLoaded layer; + + File file{ filename }; + if (!file.open(File::OpenModeFlag::ReadOnly)) { + LOG(LayerManager, Error) << "Failed to open layer: " + << strerror(-file.error()); + return layer; + } + + Span data = file.map(); + int ret = utils::elfVerifyIdent(data); + if (ret) { + LOG(LayerManager, Error) << "Layer is not an ELF file"; + return layer; + } + + Span info = utils::elfLoadSymbol(data, "layerInfo"); + if (info.size() < sizeof(LayerInfo)) { + LOG(LayerManager, Error) << "Layer has no valid info"; + return layer; + } + + void *dlHandle = dlopen(file.fileName().c_str(), RTLD_LAZY); + if (!dlHandle) { + LOG(LayerManager, Error) + << "Failed to open layer shared object: " + << dlerror(); + return layer; + } + + void *layerInfo = dlsym(dlHandle, "layerInfo"); + if (!layerInfo) { + LOG(LayerManager, Error) + << "Failed to load layerInfo from layer shared object: " + << dlerror(); + dlclose(dlHandle); + return layer; + } + + void *vtable = dlsym(dlHandle, "layerInterface"); + if (!vtable) { + LOG(LayerManager, Error) + << "Failed to load layerInterface from layer shared object: " + << dlerror(); + dlclose(dlHandle); + return layer; + } + + layer.info = static_cast(layerInfo); + layer.vtable = static_cast(vtable); + layer.dlHandle = dlHandle; + + /* + * No need to dlclose after this as the LayerLoaded deconstructor will + * handle it + */ + + /* \todo Implement this. It should come from the libcamera version */ + if (layer.info->layerAPIVersion != 1) { + LOG(LayerManager, Error) << "Layer API version mismatch"; + layer.info = nullptr; + return layer; + } + + /* \todo Document these requirements */ + if (!layer.vtable->init) { + LOG(LayerManager, Error) << "Layer doesn't implement init"; + layer.info = nullptr; + return layer; + } + + /* \todo Document these requirements */ + if (!layer.vtable->terminate) { + LOG(LayerManager, Error) << "Layer doesn't implement terminate"; + layer.info = nullptr; + return layer; + } + + /* \todo Validate the layer name. */ + + return layer; +} + +void LayerManager::bufferCompleted(const Camera *camera, Request *request, FrameBuffer *buffer) +{ + /* Reverse order because this comes from a Signal emission */ + for (auto it = executionQueue_.rbegin(); + it != executionQueue_.rend(); it++) { + if ((*it).vtable->bufferCompleted) { + void *closure = closures_.at(std::make_tuple(camera, &(*it))); + (*it).vtable->bufferCompleted(closure, request, buffer); + } + } +} + +void LayerManager::requestCompleted(const Camera *camera, Request *request) +{ + /* Reverse order because this comes from a Signal emission */ + for (auto it = executionQueue_.rbegin(); + it != executionQueue_.rend(); it++) { + if ((*it).vtable->requestCompleted) { + void *closure = closures_.at(std::make_tuple(camera, &(*it))); + (*it).vtable->requestCompleted(closure, request); + } + } +} + +void LayerManager::disconnected(const Camera *camera) +{ + /* Reverse order because this comes from a Signal emission */ + for (auto it = executionQueue_.rbegin(); + it != executionQueue_.rend(); it++) { + if ((*it).vtable->disconnected) { + void *closure = closures_.at(std::make_tuple(camera, &(*it))); + (*it).vtable->disconnected(closure); + } + } +} + +void LayerManager::acquire(const Camera *camera) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->acquire) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->acquire(closure); + } + } +} + +void LayerManager::release(const Camera *camera) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->release) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->release(closure); + } + } +} + +void LayerManager::updateProperties(const Camera *camera, + const ControlList &properties) +{ + ControlList props = properties; + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->properties) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + ControlList ret = layer.vtable->properties(closure, props); + props.merge(ret, ControlList::MergePolicy::OverwriteExisting); + } + } + properties_[camera] = props; +} + +void LayerManager::updateControls(const Camera *camera, + const ControlInfoMap &controlInfoMap) +{ + ControlInfoMap infoMap = controlInfoMap; + /* \todo Simplify this once ControlInfoMaps become easier to modify */ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->controls) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + ControlInfoMap::Map ret = layer.vtable->controls(closure, infoMap); + ControlInfoMap::Map map; + /* Merge the layer's ret later so that layers can overwrite */ + for (auto &pair : infoMap) + map.insert(pair); + for (auto &pair : ret) + map.insert(pair); + infoMap = ControlInfoMap(std::move(map), + libcamera::controls::controls); + } + } + controls_[camera] = infoMap; +} + +void LayerManager::configure(const Camera *camera, + const CameraConfiguration *config, + const ControlInfoMap &controlInfoMap) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->configure) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->configure(closure, config); + } + } + + updateControls(camera, controlInfoMap); +} + +void LayerManager::createRequest(const Camera *camera, uint64_t cookie, const Request *request) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->createRequest) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->createRequest(closure, cookie, request); + } + } +} + +void LayerManager::queueRequest(const Camera *camera, Request *request) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->queueRequest) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->queueRequest(closure, request); + } + } +} + +void LayerManager::start(const Camera *camera, const ControlList *controls) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->start) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->start(closure, controls); + } + } +} + +void LayerManager::stop(const Camera *camera) +{ + for (LayerManager::LayerLoaded &layer : executionQueue_) { + if (layer.vtable->stop) { + void *closure = closures_.at(std::make_tuple(camera, &layer)); + layer.vtable->stop(closure); + } + } +} + +} /* namespace libcamera */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 6a71b2903d27..0c2086a8399c 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -40,6 +40,7 @@ libcamera_internal_sources = files([ 'ipc_pipe.cpp', 'ipc_pipe_unixsocket.cpp', 'ipc_unixsocket.cpp', + 'layer_manager.cpp', 'mapped_framebuffer.cpp', 'matrix.cpp', 'media_device.cpp', diff --git a/src/meson.build b/src/meson.build index 8eb8f05b362f..37368b01cbf2 100644 --- a/src/meson.build +++ b/src/meson.build @@ -63,6 +63,7 @@ subdir('libcamera') subdir('android') subdir('ipa') +subdir('layer') subdir('apps')