[{"id":4121,"web_url":"https://patchwork.libcamera.org/comment/4121/","msgid":"<20200320015431.GU14585@pendragon.ideasonboard.com>","date":"2020-03-20T01:54:31","subject":"Re: [libcamera-devel] [RFC/PATCH] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"On Fri, Mar 20, 2020 at 03:16:18AM +0200, Laurent Pinchart wrote:\n> While closed-source IPA modules will always be sandboxed, open-source\n> IPA modules may be run in the main libcamera process or be sandboxed,\n> depending on platform configuration. These two models exhibit very\n> different timings, which require extensive testing with both\n> configurations.\n> \n> When run into the main libcamera process, IPA modules are executed in\n> the pipeline handler thread (which is currently a global CameraManager\n> thread). Time-consuming operations in the IPA may thus slow down the\n> pipeline handler and compromise real-time behaviour. At least some\n> pipeline handlers will thus likely spawn a thread to isolate the IPA,\n> leading to code duplication in pipeline handlers.\n> \n> Solve both issues by always proxying IPA modules. For open-source IPA\n> modules that run in the libcamera process, a new IPAProxyThread class is\n> added to run the IPA in a separate thread.\n> \n> Signed-off-by: Laurent Pinchart \n> ---\n> Hello,\n> \n> This patch is currently untested, I'm working on expanding the VIMC IPA\n> to provide unit tests. I've however decided to send it already to get\n> feedback on the idea.\n\nQuick update, thanks to Niklas' testing of the patch, we've noticed an\nissue with unmapBuffers() racing with processEvent(). I'll work on a\nfix, but the general idea holds.\n\n> src/libcamera/ipa_manager.cpp | 48 ++++-----\n> src/libcamera/proxy/ipa_proxy_thread.cpp | 130 +++++++++++++++++++++++\n> src/libcamera/proxy/meson.build | 1 +\n> 3 files changed, 152 insertions(+), 27 deletions(-)\n> create mode 100644 src/libcamera/proxy/ipa_proxy_thread.cpp\n> \n> diff --git a/src/libcamera/ipa_manager.cpp b/src/libcamera/ipa_manager.cpp\n> index bcaae3564ea1..6d23f4705065 100644\n> --- a/src/libcamera/ipa_manager.cpp\n> +++ b/src/libcamera/ipa_manager.cpp\n> @@ -12,7 +12,6 @@\n> #include \n> #include \n> \n> -#include \"ipa_context_wrapper.h\"\n> #include \"ipa_module.h\"\n> #include \"ipa_proxy.h\"\n> #include \"log.h\"\n> @@ -271,40 +270,35 @@ std::unique_ptr IPAManager::createIPA(PipelineHandler *pipe,\n> \tif (!m)\n> \t\treturn nullptr;\n> \n> -\tif (!m->isOpenSource()) {\n> -\t\tIPAProxyFactory *pf = nullptr;\n> -\t\tstd::vector &factories = IPAProxyFactory::factories();\n> -\n> -\t\tfor (IPAProxyFactory *factory : factories) {\n> -\t\t\t/* TODO: Better matching */\n> -\t\t\tif (!strcmp(factory->name().c_str(), \"IPAProxyLinux\")) {\n> -\t\t\t\tpf = factory;\n> -\t\t\t\tbreak;\n> -\t\t\t}\n> -\t\t}\n> -\n> -\t\tif (!pf) {\n> -\t\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> -\t\t\treturn nullptr;\n> -\t\t}\n> +\t/*\n> +\t * Load and run the IPA module in a thread if it is open-source, or\n> +\t * isolate it in a separate process otherwise.\n> +\t *\n> +\t * \\todo Implement a better proxy selection\n> +\t */\n> +\tconst char *proxyName = m->isOpenSource()\n> +\t\t\t ? \"IPAProxyThread\" : \"IPAProxyLinux\";\n> +\tIPAProxyFactory *pf = nullptr;\n> \n> -\t\tstd::unique_ptr proxy = pf->create(m);\n> -\t\tif (!proxy->isValid()) {\n> -\t\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> -\t\t\treturn nullptr;\n> +\tfor (IPAProxyFactory *factory : IPAProxyFactory::factories()) {\n> +\t\tif (!strcmp(factory->name().c_str(), proxyName)) {\n> +\t\t\tpf = factory;\n> +\t\t\tbreak;\n> \t\t}\n> -\n> -\t\treturn proxy;\n> \t}\n> \n> -\tif (!m->load())\n> +\tif (!pf) {\n> +\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> \t\treturn nullptr;\n> +\t}\n> \n> -\tstruct ipa_context *ctx = m->createContext();\n> -\tif (!ctx)\n> +\tstd::unique_ptr proxy = pf->create(m);\n> +\tif (!proxy->isValid()) {\n> +\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> \t\treturn nullptr;\n> +\t}\n> \n> -\treturn std::make_unique(ctx);\n> +\treturn proxy;\n> }\n> \n> } /* namespace libcamera */\n> diff --git a/src/libcamera/proxy/ipa_proxy_thread.cpp b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> new file mode 100644\n> index 000000000000..22006b3c98ee\n> --- /dev/null\n> +++ b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> @@ -0,0 +1,130 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * ipa_proxy_thread.cpp - Proxy running an Image Processing Algorithm in a thread\n> + */\n> +\n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \"ipa_context_wrapper.h\"\n> +#include \"ipa_module.h\"\n> +#include \"ipa_proxy.h\"\n> +#include \"log.h\"\n> +#include \"thread.h\"\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(IPAProxy)\n> +\n> +class IPAProxyThread : public IPAProxy, public Object\n> +{\n> +public:\n> +\tIPAProxyThread(IPAModule *ipam);\n> +\t~IPAProxyThread();\n> +\n> +\tint init() override;\n> +\tvoid configure(const std::map &streamConfig,\n> +\t\t const std::map &entityControls) override;\n> +\tvoid mapBuffers(const std::vector &buffers) override;\n> +\tvoid unmapBuffers(const std::vector &ids) override;\n> +\tvoid processEvent(const IPAOperationData &event) override;\n> +\n> +private:\n> +\tvoid queueFrameAction(unsigned int frame, const IPAOperationData &data);\n> +\n> +\t/* Helper class to invoke processEvent() in another thread. */\n> +\tclass ThreadProxy : public Object\n> +\t{\n> +\tpublic:\n> +\t\tvoid setIPA(IPAInterface *ipa)\n> +\t\t{\n> +\t\t\tipa_ = ipa;\n> +\t\t}\n> +\n> +\t\tvoid processEvent(const IPAOperationData &event)\n> +\t\t{\n> +\t\t\tipa_->processEvent(event);\n> +\t\t}\n> +\n> +\tprivate:\n> +\t\tIPAInterface *ipa_;\n> +\t};\n> +\n> +\tThread thread_;\n> +\tThreadProxy proxy_;\n> +\tstd::unique_ptr ipa_;\n> +};\n> +\n> +IPAProxyThread::IPAProxyThread(IPAModule *ipam)\n> +{\n> +\tif (!ipam->load())\n> +\t\treturn;\n> +\n> +\tstruct ipa_context *ctx = ipam->createContext();\n> +\tif (!ctx) {\n> +\t\tLOG(IPAProxy, Error)\n> +\t\t\t<< \"Failed to create IPA context for \" << ipam->path();\n> +\t\treturn;\n> +\t}\n> +\n> +\tipa_ = std::make_unique(ctx);\n> +\tproxy_.setIPA(ipa_.get());\n> +\n> +\t/*\n> +\t * Proxy the queueFrameAction signal to dispatch it in the caller's\n> +\t * thread.\n> +\t */\n> +\tipa_->queueFrameAction.connect(this, &IPAProxyThread::queueFrameAction);\n> +\n> +\tvalid_ = true;\n> +}\n> +\n> +IPAProxyThread::~IPAProxyThread()\n> +{\n> +\tthread_.exit();\n> +\tthread_.wait();\n> +}\n> +\n> +int IPAProxyThread::init()\n> +{\n> +\tthread_.start();\n> +\tproxy_.moveToThread(&thread_);\n> +\n> +\treturn ipa_->init();\n> +}\n> +\n> +void IPAProxyThread::configure(const std::map &streamConfig,\n> +\t\t\t const std::map &entityControls)\n> +{\n> +\tipa_->configure(streamConfig, entityControls);\n> +}\n> +\n> +void IPAProxyThread::mapBuffers(const std::vector &buffers)\n> +{\n> +\tipa_->mapBuffers(buffers);\n> +}\n> +\n> +void IPAProxyThread::unmapBuffers(const std::vector &ids)\n> +{\n> +\tipa_->unmapBuffers(ids);\n> +}\n> +\n> +void IPAProxyThread::processEvent(const IPAOperationData &event)\n> +{\n> +\t/* Dispatch the processEvent() call to the thread. */\n> +\tproxy_.invokeMethod(&ThreadProxy::processEvent, ConnectionTypeQueued,\n> +\t\t\t event);\n> +}\n> +\n> +void IPAProxyThread::queueFrameAction(unsigned int frame, const IPAOperationData &data)\n> +{\n> +\tIPAInterface::queueFrameAction.emit(frame, data);\n> +}\n> +\n> +REGISTER_IPA_PROXY(IPAProxyThread)\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/proxy/meson.build b/src/libcamera/proxy/meson.build\n> index efc113230217..6c00d5f30ad2 100644\n> --- a/src/libcamera/proxy/meson.build\n> +++ b/src/libcamera/proxy/meson.build\n> @@ -1,3 +1,4 @@\n> libcamera_sources += files([\n> 'ipa_proxy_linux.cpp',\n> + 'ipa_proxy_thread.cpp',\n> ])\n> -- \n> Regards,\n> \n> Laurent Pinchart\n> \n> _______________________________________________\n> libcamera-devel mailing list\n> libcamera-devel@lists.libcamera.org\n> https://lists.libcamera.org/listinfo/libcamera-devel","headers":{"Return-Path":"","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BF70560418\n\tfor ;\n\tFri, 20 Mar 2020 02:54:37 +0100 (CET)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 34DB1504\n\tfor ;\n\tFri, 20 Mar 2020 02:54:37 +0100 (CET)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1584669277;\n\tbh=UMpkF+KTKAj8sRvJILYJy5eE/NGmHEGJgEFpIJfrS8M=;\n\th=Date:From:To:Subject:References:In-Reply-To:From;\n\tb=UrnxaXeyMWdJdH5QtS+jtUGebCXsfUUnC03x1EWuGQ86H/iwUGRl8OyBOG9KXbX5w\n\tuUBUcNFbIoX9Xznchp+RNXTGuthN1hXplg30B5IVcR9bh6FU8lz6gtmmoNwNwO8rrk\n\tzuI/P4FHphbo+/lDLTOhN5myJbPvtETwUbnvl120=","Date":"Fri, 20 Mar 2020 03:54:31 +0200","From":"Laurent Pinchart ","To":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200320015431.GU14585@pendragon.ideasonboard.com>","References":"<20200320011618.13331-1-laurent.pinchart@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20200320011618.13331-1-laurent.pinchart@ideasonboard.com>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [RFC/PATCH] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Fri, 20 Mar 2020 01:54:38 -0000"}}]