[{"id":4324,"web_url":"https://patchwork.libcamera.org/comment/4324/","msgid":"<20200326213726.GE20581@pendragon.ideasonboard.com>","date":"2020-03-26T21:37:26","subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas,\n\nThank you for my patch :-)\n\nOn Thu, Mar 26, 2020 at 05:08:19PM +0100, Niklas Söderlund wrote:\n> From: Laurent Pinchart \n> \n> While closed-source IPA modules will always be sandboxed, open-source\n> IPA modules may be run in the main libcamera process or be sandboxed,\n> depending on platform configuration. These two models exhibit very\n> different timings, which require extensive testing with both\n> configurations.\n> \n> When run into the main libcamera process, IPA modules are executed in\n> the pipeline handler thread (which is currently a global CameraManager\n> thread). Time-consuming operations in the IPA may thus slow down the\n> pipeline handler and compromise real-time behaviour. At least some\n> pipeline handlers will thus likely spawn a thread to isolate the IPA,\n> leading to code duplication in pipeline handlers.\n> \n> Solve both issues by always proxying IPA modules. For open-source IPA\n> modules that run in the libcamera process, a new IPAProxyThread class is\n> added to run the IPA in a separate thread.\n> \n> Signed-off-by: Laurent Pinchart \n> [Niklas: Move thread start/stop of thread into start()/stop()]\n> Signed-off-by: Niklas Söderlund \n> ---\n> src/libcamera/ipa_manager.cpp | 48 ++++----\n> src/libcamera/proxy/ipa_proxy_thread.cpp | 144 +++++++++++++++++++++++\n> src/libcamera/proxy/meson.build | 1 +\n> 3 files changed, 166 insertions(+), 27 deletions(-)\n> create mode 100644 src/libcamera/proxy/ipa_proxy_thread.cpp\n> \n> diff --git a/src/libcamera/ipa_manager.cpp b/src/libcamera/ipa_manager.cpp\n> index bcaae3564ea14e07..6d23f470506561b8 100644\n> --- a/src/libcamera/ipa_manager.cpp\n> +++ b/src/libcamera/ipa_manager.cpp\n> @@ -12,7 +12,6 @@\n> #include \n> #include \n> \n> -#include \"ipa_context_wrapper.h\"\n> #include \"ipa_module.h\"\n> #include \"ipa_proxy.h\"\n> #include \"log.h\"\n> @@ -271,40 +270,35 @@ std::unique_ptr IPAManager::createIPA(PipelineHandler *pipe,\n> \tif (!m)\n> \t\treturn nullptr;\n> \n> -\tif (!m->isOpenSource()) {\n> -\t\tIPAProxyFactory *pf = nullptr;\n> -\t\tstd::vector &factories = IPAProxyFactory::factories();\n> +\t/*\n> +\t * Load and run the IPA module in a thread if it is open-source, or\n> +\t * isolate it in a separate process otherwise.\n> +\t *\n> +\t * \\todo Implement a better proxy selection\n> +\t */\n> +\tconst char *proxyName = m->isOpenSource()\n> +\t\t\t ? \"IPAProxyThread\" : \"IPAProxyLinux\";\n> +\tIPAProxyFactory *pf = nullptr;\n> \n> -\t\tfor (IPAProxyFactory *factory : factories) {\n> -\t\t\t/* TODO: Better matching */\n> -\t\t\tif (!strcmp(factory->name().c_str(), \"IPAProxyLinux\")) {\n> -\t\t\t\tpf = factory;\n> -\t\t\t\tbreak;\n> -\t\t\t}\n> +\tfor (IPAProxyFactory *factory : IPAProxyFactory::factories()) {\n> +\t\tif (!strcmp(factory->name().c_str(), proxyName)) {\n> +\t\t\tpf = factory;\n> +\t\t\tbreak;\n> \t\t}\n> -\n> -\t\tif (!pf) {\n> -\t\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> -\t\t\treturn nullptr;\n> -\t\t}\n> -\n> -\t\tstd::unique_ptr proxy = pf->create(m);\n> -\t\tif (!proxy->isValid()) {\n> -\t\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> -\t\t\treturn nullptr;\n> -\t\t}\n> -\n> -\t\treturn proxy;\n> \t}\n> \n> -\tif (!m->load())\n> +\tif (!pf) {\n> +\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> \t\treturn nullptr;\n> +\t}\n> \n> -\tstruct ipa_context *ctx = m->createContext();\n> -\tif (!ctx)\n> +\tstd::unique_ptr proxy = pf->create(m);\n> +\tif (!proxy->isValid()) {\n> +\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> \t\treturn nullptr;\n> +\t}\n> \n> -\treturn std::make_unique(ctx);\n> +\treturn proxy;\n> }\n> \n> } /* namespace libcamera */\n> diff --git a/src/libcamera/proxy/ipa_proxy_thread.cpp b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> new file mode 100644\n> index 0000000000000000..6719bcdda43f3647\n> --- /dev/null\n> +++ b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> @@ -0,0 +1,144 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * ipa_proxy_thread.cpp - Proxy running an Image Processing Algorithm in a thread\n> + */\n> +\n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \"ipa_context_wrapper.h\"\n> +#include \"ipa_module.h\"\n> +#include \"ipa_proxy.h\"\n> +#include \"log.h\"\n> +#include \"thread.h\"\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(IPAProxy)\n> +\n> +class IPAProxyThread : public IPAProxy, public Object\n> +{\n> +public:\n> +\tIPAProxyThread(IPAModule *ipam);\n> +\n> +\tint init() override;\n> +\tint start() override;\n> +\tvoid stop() override;\n> +\n> +\tvoid configure(const std::map &streamConfig,\n> +\t\t const std::map &entityControls) override;\n> +\tvoid mapBuffers(const std::vector &buffers) override;\n> +\tvoid unmapBuffers(const std::vector &ids) override;\n> +\tvoid processEvent(const IPAOperationData &event) override;\n> +\n> +private:\n> +\tvoid queueFrameAction(unsigned int frame, const IPAOperationData &data);\n> +\n> +\t/* Helper class to invoke processEvent() in another thread. */\n> +\tclass ThreadProxy : public Object\n> +\t{\n> +\tpublic:\n> +\t\tvoid setIPA(IPAInterface *ipa)\n> +\t\t{\n> +\t\t\tipa_ = ipa;\n> +\t\t}\n> +\n> +\t\tvoid stop()\n> +\t\t{\n> +\t\t\tipa_->stop();\n> +\t\t}\n> +\n> +\t\tvoid processEvent(const IPAOperationData &event)\n> +\t\t{\n> +\t\t\tipa_->processEvent(event);\n> +\t\t}\n> +\n> +\tprivate:\n> +\t\tIPAInterface *ipa_;\n> +\t};\n> +\n> +\tThread thread_;\n> +\tThreadProxy proxy_;\n> +\tstd::unique_ptr ipa_;\n> +};\n> +\n> +IPAProxyThread::IPAProxyThread(IPAModule *ipam)\n> +{\n> +\tif (!ipam->load())\n> +\t\treturn;\n> +\n> +\tstruct ipa_context *ctx = ipam->createContext();\n> +\tif (!ctx) {\n> +\t\tLOG(IPAProxy, Error)\n> +\t\t\t<< \"Failed to create IPA context for \" << ipam->path();\n> +\t\treturn;\n> +\t}\n> +\n> +\tipa_ = std::make_unique(ctx);\n> +\tproxy_.setIPA(ipa_.get());\n> +\n> +\t/*\n> +\t * Proxy the queueFrameAction signal to dispatch it in the caller's\n> +\t * thread.\n> +\t */\n> +\tipa_->queueFrameAction.connect(this, &IPAProxyThread::queueFrameAction);\n> +\n> +\tvalid_ = true;\n> +}\n> +\n> +int IPAProxyThread::init()\n> +{\n> +\treturn ipa_->init();\n> +}\n> +\n> +int IPAProxyThread::start()\n> +{\n> +\tthread_.start();\n> +\tproxy_.moveToThread(&thread_);\n\nIf we stop and restart, we will move the proxy to the thread twice. It\nshouldn't be an issue, but I wonder if we should move the proxy at init\ntime, or even in the constructor. I *think* that moving and object to a\nthread that hasn't been started yet is fine, but please test it :-)\n\n> +\n> +\treturn ipa_->start();\n\nShouldn't start be called in the target thread, as the thread is already\nrunning ?\n\n> +}\n> +\n> +void IPAProxyThread::stop()\n> +{\n> +\tproxy_.invokeMethod(&ThreadProxy::stop, ConnectionTypeBlocking);\n> +\n> +\tthread_.exit();\n> +\tthread_.wait();\n> +}\n> +\n> +void IPAProxyThread::configure(const std::map &streamConfig,\n> +\t\t\t const std::map &entityControls)\n> +{\n> +\tipa_->configure(streamConfig, entityControls);\n> +}\n> +\n> +void IPAProxyThread::mapBuffers(const std::vector &buffers)\n> +{\n> +\tipa_->mapBuffers(buffers);\n> +}\n> +\n> +void IPAProxyThread::unmapBuffers(const std::vector &ids)\n> +{\n> +\tipa_->unmapBuffers(ids);\n> +}\n> +\n> +void IPAProxyThread::processEvent(const IPAOperationData &event)\n> +{\n> +\t/* Dispatch the processEvent() call to the thread. */\n> +\tproxy_.invokeMethod(&ThreadProxy::processEvent, ConnectionTypeQueued,\n> +\t\t\t event);\n> +}\n> +\n> +void IPAProxyThread::queueFrameAction(unsigned int frame, const IPAOperationData &data)\n> +{\n\nInstead of blocking frame actions in the pipeline handlers, would it\nmake sense to block them here when we're stopping ? This function is\ncalled in the pipeline handler thread, and so won't race with stop().\n\n> +\tIPAInterface::queueFrameAction.emit(frame, data);\n> +}\n> +\n> +REGISTER_IPA_PROXY(IPAProxyThread)\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/proxy/meson.build b/src/libcamera/proxy/meson.build\n> index efc1132302176f63..6c00d5f30ad2e5f0 100644\n> --- a/src/libcamera/proxy/meson.build\n> +++ b/src/libcamera/proxy/meson.build\n> @@ -1,3 +1,4 @@\n> libcamera_sources += files([\n> 'ipa_proxy_linux.cpp',\n> + 'ipa_proxy_thread.cpp',\n> ])","headers":{"Return-Path":"","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 810B760412\n\tfor ;\n\tThu, 26 Mar 2020 22:37:30 +0100 (CET)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id E49EE2DC;\n\tThu, 26 Mar 2020 22:37:29 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"IM8uajNx\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1585258650;\n\tbh=6PcznD7Fzec9KiB0FwUChuRbBVs4xpJTVRLktooc9Do=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=IM8uajNxtoNBGmkPtt3Ph+Ch08RTmj5Samf6ZkLDP7kLONP7H08gjbLqd7VURVXEU\n\tJL6PuUhS9RTvfbM1RQOv4uNxb8FvRJPfYCa5l0iO2kg48OTvnZ8yzJ7qWNVUXGeUz8\n\t72XLOHZqXThWjbaxf1I3h7InuRqM1E8hJHaiYbos=","Date":"Thu, 26 Mar 2020 23:37:26 +0200","From":"Laurent Pinchart ","To":"Niklas =?utf-8?q?S=C3=B6derlund?= ","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200326213726.GE20581@pendragon.ideasonboard.com>","References":"<20200326160819.4088361-1-niklas.soderlund@ragnatech.se>\n\t<20200326160819.4088361-8-niklas.soderlund@ragnatech.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200326160819.4088361-8-niklas.soderlund@ragnatech.se>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Thu, 26 Mar 2020 21:37:30 -0000"}},{"id":4327,"web_url":"https://patchwork.libcamera.org/comment/4327/","msgid":"<20200326214843.GB1844461@oden.dyn.berto.se>","date":"2020-03-26T21:48:43","subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","submitter":{"id":5,"url":"https://patchwork.libcamera.org/api/people/5/","name":"Niklas Söderlund","email":"niklas.soderlund@ragnatech.se"},"content":"Hi Laurent,\n\nThanks for your feedback.\n\nOn 2020-03-26 23:37:26 +0200, Laurent Pinchart wrote:\n> Hi Niklas,\n> \n> Thank you for my patch :-)\n> \n> On Thu, Mar 26, 2020 at 05:08:19PM +0100, Niklas Söderlund wrote:\n> > From: Laurent Pinchart \n> > \n> > While closed-source IPA modules will always be sandboxed, open-source\n> > IPA modules may be run in the main libcamera process or be sandboxed,\n> > depending on platform configuration. These two models exhibit very\n> > different timings, which require extensive testing with both\n> > configurations.\n> > \n> > When run into the main libcamera process, IPA modules are executed in\n> > the pipeline handler thread (which is currently a global CameraManager\n> > thread). Time-consuming operations in the IPA may thus slow down the\n> > pipeline handler and compromise real-time behaviour. At least some\n> > pipeline handlers will thus likely spawn a thread to isolate the IPA,\n> > leading to code duplication in pipeline handlers.\n> > \n> > Solve both issues by always proxying IPA modules. For open-source IPA\n> > modules that run in the libcamera process, a new IPAProxyThread class is\n> > added to run the IPA in a separate thread.\n> > \n> > Signed-off-by: Laurent Pinchart \n> > [Niklas: Move thread start/stop of thread into start()/stop()]\n> > Signed-off-by: Niklas Söderlund \n> > ---\n> > src/libcamera/ipa_manager.cpp | 48 ++++----\n> > src/libcamera/proxy/ipa_proxy_thread.cpp | 144 +++++++++++++++++++++++\n> > src/libcamera/proxy/meson.build | 1 +\n> > 3 files changed, 166 insertions(+), 27 deletions(-)\n> > create mode 100644 src/libcamera/proxy/ipa_proxy_thread.cpp\n> > \n> > diff --git a/src/libcamera/ipa_manager.cpp b/src/libcamera/ipa_manager.cpp\n> > index bcaae3564ea14e07..6d23f470506561b8 100644\n> > --- a/src/libcamera/ipa_manager.cpp\n> > +++ b/src/libcamera/ipa_manager.cpp\n> > @@ -12,7 +12,6 @@\n> > #include \n> > #include \n> > \n> > -#include \"ipa_context_wrapper.h\"\n> > #include \"ipa_module.h\"\n> > #include \"ipa_proxy.h\"\n> > #include \"log.h\"\n> > @@ -271,40 +270,35 @@ std::unique_ptr IPAManager::createIPA(PipelineHandler *pipe,\n> > \tif (!m)\n> > \t\treturn nullptr;\n> > \n> > -\tif (!m->isOpenSource()) {\n> > -\t\tIPAProxyFactory *pf = nullptr;\n> > -\t\tstd::vector &factories = IPAProxyFactory::factories();\n> > +\t/*\n> > +\t * Load and run the IPA module in a thread if it is open-source, or\n> > +\t * isolate it in a separate process otherwise.\n> > +\t *\n> > +\t * \\todo Implement a better proxy selection\n> > +\t */\n> > +\tconst char *proxyName = m->isOpenSource()\n> > +\t\t\t ? \"IPAProxyThread\" : \"IPAProxyLinux\";\n> > +\tIPAProxyFactory *pf = nullptr;\n> > \n> > -\t\tfor (IPAProxyFactory *factory : factories) {\n> > -\t\t\t/* TODO: Better matching */\n> > -\t\t\tif (!strcmp(factory->name().c_str(), \"IPAProxyLinux\")) {\n> > -\t\t\t\tpf = factory;\n> > -\t\t\t\tbreak;\n> > -\t\t\t}\n> > +\tfor (IPAProxyFactory *factory : IPAProxyFactory::factories()) {\n> > +\t\tif (!strcmp(factory->name().c_str(), proxyName)) {\n> > +\t\t\tpf = factory;\n> > +\t\t\tbreak;\n> > \t\t}\n> > -\n> > -\t\tif (!pf) {\n> > -\t\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> > -\t\t\treturn nullptr;\n> > -\t\t}\n> > -\n> > -\t\tstd::unique_ptr proxy = pf->create(m);\n> > -\t\tif (!proxy->isValid()) {\n> > -\t\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> > -\t\t\treturn nullptr;\n> > -\t\t}\n> > -\n> > -\t\treturn proxy;\n> > \t}\n> > \n> > -\tif (!m->load())\n> > +\tif (!pf) {\n> > +\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> > \t\treturn nullptr;\n> > +\t}\n> > \n> > -\tstruct ipa_context *ctx = m->createContext();\n> > -\tif (!ctx)\n> > +\tstd::unique_ptr proxy = pf->create(m);\n> > +\tif (!proxy->isValid()) {\n> > +\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> > \t\treturn nullptr;\n> > +\t}\n> > \n> > -\treturn std::make_unique(ctx);\n> > +\treturn proxy;\n> > }\n> > \n> > } /* namespace libcamera */\n> > diff --git a/src/libcamera/proxy/ipa_proxy_thread.cpp b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> > new file mode 100644\n> > index 0000000000000000..6719bcdda43f3647\n> > --- /dev/null\n> > +++ b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> > @@ -0,0 +1,144 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2020, Google Inc.\n> > + *\n> > + * ipa_proxy_thread.cpp - Proxy running an Image Processing Algorithm in a thread\n> > + */\n> > +\n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \"ipa_context_wrapper.h\"\n> > +#include \"ipa_module.h\"\n> > +#include \"ipa_proxy.h\"\n> > +#include \"log.h\"\n> > +#include \"thread.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +LOG_DECLARE_CATEGORY(IPAProxy)\n> > +\n> > +class IPAProxyThread : public IPAProxy, public Object\n> > +{\n> > +public:\n> > +\tIPAProxyThread(IPAModule *ipam);\n> > +\n> > +\tint init() override;\n> > +\tint start() override;\n> > +\tvoid stop() override;\n> > +\n> > +\tvoid configure(const std::map &streamConfig,\n> > +\t\t const std::map &entityControls) override;\n> > +\tvoid mapBuffers(const std::vector &buffers) override;\n> > +\tvoid unmapBuffers(const std::vector &ids) override;\n> > +\tvoid processEvent(const IPAOperationData &event) override;\n> > +\n> > +private:\n> > +\tvoid queueFrameAction(unsigned int frame, const IPAOperationData &data);\n> > +\n> > +\t/* Helper class to invoke processEvent() in another thread. */\n> > +\tclass ThreadProxy : public Object\n> > +\t{\n> > +\tpublic:\n> > +\t\tvoid setIPA(IPAInterface *ipa)\n> > +\t\t{\n> > +\t\t\tipa_ = ipa;\n> > +\t\t}\n> > +\n> > +\t\tvoid stop()\n> > +\t\t{\n> > +\t\t\tipa_->stop();\n> > +\t\t}\n> > +\n> > +\t\tvoid processEvent(const IPAOperationData &event)\n> > +\t\t{\n> > +\t\t\tipa_->processEvent(event);\n> > +\t\t}\n> > +\n> > +\tprivate:\n> > +\t\tIPAInterface *ipa_;\n> > +\t};\n> > +\n> > +\tThread thread_;\n> > +\tThreadProxy proxy_;\n> > +\tstd::unique_ptr ipa_;\n> > +};\n> > +\n> > +IPAProxyThread::IPAProxyThread(IPAModule *ipam)\n> > +{\n> > +\tif (!ipam->load())\n> > +\t\treturn;\n> > +\n> > +\tstruct ipa_context *ctx = ipam->createContext();\n> > +\tif (!ctx) {\n> > +\t\tLOG(IPAProxy, Error)\n> > +\t\t\t<< \"Failed to create IPA context for \" << ipam->path();\n> > +\t\treturn;\n> > +\t}\n> > +\n> > +\tipa_ = std::make_unique(ctx);\n> > +\tproxy_.setIPA(ipa_.get());\n> > +\n> > +\t/*\n> > +\t * Proxy the queueFrameAction signal to dispatch it in the caller's\n> > +\t * thread.\n> > +\t */\n> > +\tipa_->queueFrameAction.connect(this, &IPAProxyThread::queueFrameAction);\n> > +\n> > +\tvalid_ = true;\n> > +}\n> > +\n> > +int IPAProxyThread::init()\n> > +{\n> > +\treturn ipa_->init();\n> > +}\n> > +\n> > +int IPAProxyThread::start()\n> > +{\n> > +\tthread_.start();\n> > +\tproxy_.moveToThread(&thread_);\n> \n> If we stop and restart, we will move the proxy to the thread twice. It\n> shouldn't be an issue, but I wonder if we should move the proxy at init\n> time, or even in the constructor. I *think* that moving and object to a\n> thread that hasn't been started yet is fine, but please test it :-)\n\nI will test it.\n\n> \n> > +\n> > +\treturn ipa_->start();\n> \n> Shouldn't start be called in the target thread, as the thread is already\n> running ?\n\nGood point.\n\n> \n> > +}\n> > +\n> > +void IPAProxyThread::stop()\n> > +{\n> > +\tproxy_.invokeMethod(&ThreadProxy::stop, ConnectionTypeBlocking);\n> > +\n> > +\tthread_.exit();\n> > +\tthread_.wait();\n> > +}\n> > +\n> > +void IPAProxyThread::configure(const std::map &streamConfig,\n> > +\t\t\t const std::map &entityControls)\n> > +{\n> > +\tipa_->configure(streamConfig, entityControls);\n> > +}\n> > +\n> > +void IPAProxyThread::mapBuffers(const std::vector &buffers)\n> > +{\n> > +\tipa_->mapBuffers(buffers);\n> > +}\n> > +\n> > +void IPAProxyThread::unmapBuffers(const std::vector &ids)\n> > +{\n> > +\tipa_->unmapBuffers(ids);\n> > +}\n> > +\n> > +void IPAProxyThread::processEvent(const IPAOperationData &event)\n> > +{\n> > +\t/* Dispatch the processEvent() call to the thread. */\n> > +\tproxy_.invokeMethod(&ThreadProxy::processEvent, ConnectionTypeQueued,\n> > +\t\t\t event);\n> > +}\n> > +\n> > +void IPAProxyThread::queueFrameAction(unsigned int frame, const IPAOperationData &data)\n> > +{\n> \n> Instead of blocking frame actions in the pipeline handlers, would it\n> make sense to block them here when we're stopping ? This function is\n> called in the pipeline handler thread, and so won't race with stop().\n\nI think that could be neat. I thought about it but decided against it as \nit could create discrepancies in behavior for different IPAProxy \nimplementations.\n\nBut since we will have relative few IPAProxy implementations and \nhopefully many pipelines I think it could make sens to move it here, I \nwill do so for next version, if you don't object.\n\n> \n> > +\tIPAInterface::queueFrameAction.emit(frame, data);\n> > +}\n> > +\n> > +REGISTER_IPA_PROXY(IPAProxyThread)\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/libcamera/proxy/meson.build b/src/libcamera/proxy/meson.build\n> > index efc1132302176f63..6c00d5f30ad2e5f0 100644\n> > --- a/src/libcamera/proxy/meson.build\n> > +++ b/src/libcamera/proxy/meson.build\n> > @@ -1,3 +1,4 @@\n> > libcamera_sources += files([\n> > 'ipa_proxy_linux.cpp',\n> > + 'ipa_proxy_thread.cpp',\n> > ])\n> \n> -- \n> Regards,\n> \n> Laurent Pinchart","headers":{"Return-Path":"","Received":["from mail-lj1-x231.google.com (mail-lj1-x231.google.com\n\t[IPv6:2a00:1450:4864:20::231])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 091A160412\n\tfor ;\n\tThu, 26 Mar 2020 22:48:45 +0100 (CET)","by mail-lj1-x231.google.com with SMTP id w1so8095264ljh.5\n\tfor ;\n\tThu, 26 Mar 2020 14:48:44 -0700 (PDT)","from localhost (h-200-138.A463.priv.bahnhof.se. [176.10.200.138])\n\tby smtp.gmail.com with ESMTPSA id\n\tm6sm1958185ljj.78.2020.03.26.14.48.43\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tThu, 26 Mar 2020 14:48:43 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected)\n\theader.d=ragnatech-se.20150623.gappssmtp.com\n\theader.i=@ragnatech-se.20150623.gappssmtp.com header.b=\"NW2GfiHT\"; \n\tdkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=ragnatech-se.20150623.gappssmtp.com; s=20150623;\n\th=date:from:to:cc:subject:message-id:references:mime-version\n\t:content-disposition:content-transfer-encoding:in-reply-to;\n\tbh=D0r0b85DyVxFhMBmTw55P9BKvMZEnjBIuO5AZ5u2VHw=;\n\tb=NW2GfiHTSxFT+OK/+wZp5lhvwXySGgHFtrptLDscfBkwVBzueN07vmMrAshSrQkZow\n\t6A5Yl2XZZYNcSxgXGKGqzKNbD1fq2CcSf6+qrqKVgXmhuf41XEuG29yWP25cAp3HSOWn\n\tTGesK5BNMmnZLYSeJt0WZqXdsBlN9hUTykx1LgqSNPWqVhIVWjsl+sMeo7XkjrGIuh66\n\tdH9nIZlet24H/UzEPtLpvumQTeYfZTI7vi810iEj6/hptlKFeqA+BeK0n96xUD57wB/o\n\tCPCCptJ6dE6ENuCCQdzKH2KjM3EXohgaYJbPEKEPt2x1uvnIq0SaqasuQQEuToYjoPWU\n\tWRJw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:date:from:to:cc:subject:message-id:references\n\t:mime-version:content-disposition:content-transfer-encoding\n\t:in-reply-to;\n\tbh=D0r0b85DyVxFhMBmTw55P9BKvMZEnjBIuO5AZ5u2VHw=;\n\tb=KqutxWtmJUUlBlvFj9jfWnsZ4b5JVg40j8d20YsIlo5ibXcQodkfJDYRtJHA6mLq8P\n\tdxDMvcELUPIgYu3jbKt5/0nruePjfACsckGxE2KeZpHVXY4mHprzAB9cvWMY7tuC7h/v\n\tCLvsvK8489GZ/sX52He3f8Ggxz+3pfEZFoB4gLGolGwmvE2GQRMTw0+qbecD6QWk+xJi\n\tvJzFLeZnrY0NP10gB3rz59JccWWgZvtax0RgJSsB5o0T+rYOL/I3oyJU3kssg6pwNeag\n\t1UuIySmW5U2/ZJO4PPC4b9TrLNvbiaM1GsmZI6aP93RpbyhghJKFyKwHDBtH/ItgtlXB\n\t36Eg==","X-Gm-Message-State":"ANhLgQ3TNzGS3tvqPguhGuYEjuvIgBWlzgTyJ7XeuadKPeMdcMrODvSa\n\tHWJsajmYYxbkaOCWcZCP3ZF1M5/3HvM=","X-Google-Smtp-Source":"APiQypJGxm3DhEJR9R0S2YoEbfM5qxfbwrZXCyihvDoAeukS9MSq72wnHBF1kyTuFdCHd5QKiTzwiA==","X-Received":"by 2002:a2e:8015:: with SMTP id\n\tj21mr6166683ljg.165.1585259324188; \n\tThu, 26 Mar 2020 14:48:44 -0700 (PDT)","Date":"Thu, 26 Mar 2020 22:48:43 +0100","From":"Niklas =?iso-8859-1?q?S=F6derlund?= ","To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200326214843.GB1844461@oden.dyn.berto.se>","References":"<20200326160819.4088361-1-niklas.soderlund@ragnatech.se>\n\t<20200326160819.4088361-8-niklas.soderlund@ragnatech.se>\n\t<20200326213726.GE20581@pendragon.ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=iso-8859-1","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200326213726.GE20581@pendragon.ideasonboard.com>","Subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Thu, 26 Mar 2020 21:48:45 -0000"}},{"id":4328,"web_url":"https://patchwork.libcamera.org/comment/4328/","msgid":"<20200326215305.GG20581@pendragon.ideasonboard.com>","date":"2020-03-26T21:53:05","subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas,\n\nOn Thu, Mar 26, 2020 at 10:48:43PM +0100, Niklas Söderlund wrote:\n> On 2020-03-26 23:37:26 +0200, Laurent Pinchart wrote:\n> > On Thu, Mar 26, 2020 at 05:08:19PM +0100, Niklas Söderlund wrote:\n> > > From: Laurent Pinchart \n> > > \n> > > While closed-source IPA modules will always be sandboxed, open-source\n> > > IPA modules may be run in the main libcamera process or be sandboxed,\n> > > depending on platform configuration. These two models exhibit very\n> > > different timings, which require extensive testing with both\n> > > configurations.\n> > > \n> > > When run into the main libcamera process, IPA modules are executed in\n> > > the pipeline handler thread (which is currently a global CameraManager\n> > > thread). Time-consuming operations in the IPA may thus slow down the\n> > > pipeline handler and compromise real-time behaviour. At least some\n> > > pipeline handlers will thus likely spawn a thread to isolate the IPA,\n> > > leading to code duplication in pipeline handlers.\n> > > \n> > > Solve both issues by always proxying IPA modules. For open-source IPA\n> > > modules that run in the libcamera process, a new IPAProxyThread class is\n> > > added to run the IPA in a separate thread.\n> > > \n> > > Signed-off-by: Laurent Pinchart \n> > > [Niklas: Move thread start/stop of thread into start()/stop()]\n> > > Signed-off-by: Niklas Söderlund \n> > > ---\n> > > src/libcamera/ipa_manager.cpp | 48 ++++----\n> > > src/libcamera/proxy/ipa_proxy_thread.cpp | 144 +++++++++++++++++++++++\n> > > src/libcamera/proxy/meson.build | 1 +\n> > > 3 files changed, 166 insertions(+), 27 deletions(-)\n> > > create mode 100644 src/libcamera/proxy/ipa_proxy_thread.cpp\n> > > \n> > > diff --git a/src/libcamera/ipa_manager.cpp b/src/libcamera/ipa_manager.cpp\n> > > index bcaae3564ea14e07..6d23f470506561b8 100644\n> > > --- a/src/libcamera/ipa_manager.cpp\n> > > +++ b/src/libcamera/ipa_manager.cpp\n> > > @@ -12,7 +12,6 @@\n> > > #include \n> > > #include \n> > > \n> > > -#include \"ipa_context_wrapper.h\"\n> > > #include \"ipa_module.h\"\n> > > #include \"ipa_proxy.h\"\n> > > #include \"log.h\"\n> > > @@ -271,40 +270,35 @@ std::unique_ptr IPAManager::createIPA(PipelineHandler *pipe,\n> > > \tif (!m)\n> > > \t\treturn nullptr;\n> > > \n> > > -\tif (!m->isOpenSource()) {\n> > > -\t\tIPAProxyFactory *pf = nullptr;\n> > > -\t\tstd::vector &factories = IPAProxyFactory::factories();\n> > > +\t/*\n> > > +\t * Load and run the IPA module in a thread if it is open-source, or\n> > > +\t * isolate it in a separate process otherwise.\n> > > +\t *\n> > > +\t * \\todo Implement a better proxy selection\n> > > +\t */\n> > > +\tconst char *proxyName = m->isOpenSource()\n> > > +\t\t\t ? \"IPAProxyThread\" : \"IPAProxyLinux\";\n> > > +\tIPAProxyFactory *pf = nullptr;\n> > > \n> > > -\t\tfor (IPAProxyFactory *factory : factories) {\n> > > -\t\t\t/* TODO: Better matching */\n> > > -\t\t\tif (!strcmp(factory->name().c_str(), \"IPAProxyLinux\")) {\n> > > -\t\t\t\tpf = factory;\n> > > -\t\t\t\tbreak;\n> > > -\t\t\t}\n> > > +\tfor (IPAProxyFactory *factory : IPAProxyFactory::factories()) {\n> > > +\t\tif (!strcmp(factory->name().c_str(), proxyName)) {\n> > > +\t\t\tpf = factory;\n> > > +\t\t\tbreak;\n> > > \t\t}\n> > > -\n> > > -\t\tif (!pf) {\n> > > -\t\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> > > -\t\t\treturn nullptr;\n> > > -\t\t}\n> > > -\n> > > -\t\tstd::unique_ptr proxy = pf->create(m);\n> > > -\t\tif (!proxy->isValid()) {\n> > > -\t\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> > > -\t\t\treturn nullptr;\n> > > -\t\t}\n> > > -\n> > > -\t\treturn proxy;\n> > > \t}\n> > > \n> > > -\tif (!m->load())\n> > > +\tif (!pf) {\n> > > +\t\tLOG(IPAManager, Error) << \"Failed to get proxy factory\";\n> > > \t\treturn nullptr;\n> > > +\t}\n> > > \n> > > -\tstruct ipa_context *ctx = m->createContext();\n> > > -\tif (!ctx)\n> > > +\tstd::unique_ptr proxy = pf->create(m);\n> > > +\tif (!proxy->isValid()) {\n> > > +\t\tLOG(IPAManager, Error) << \"Failed to load proxy\";\n> > > \t\treturn nullptr;\n> > > +\t}\n> > > \n> > > -\treturn std::make_unique(ctx);\n> > > +\treturn proxy;\n> > > }\n> > > \n> > > } /* namespace libcamera */\n> > > diff --git a/src/libcamera/proxy/ipa_proxy_thread.cpp b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> > > new file mode 100644\n> > > index 0000000000000000..6719bcdda43f3647\n> > > --- /dev/null\n> > > +++ b/src/libcamera/proxy/ipa_proxy_thread.cpp\n> > > @@ -0,0 +1,144 @@\n> > > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > > +/*\n> > > + * Copyright (C) 2020, Google Inc.\n> > > + *\n> > > + * ipa_proxy_thread.cpp - Proxy running an Image Processing Algorithm in a thread\n> > > + */\n> > > +\n> > > +#include \n> > > +\n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \"ipa_context_wrapper.h\"\n> > > +#include \"ipa_module.h\"\n> > > +#include \"ipa_proxy.h\"\n> > > +#include \"log.h\"\n> > > +#include \"thread.h\"\n> > > +\n> > > +namespace libcamera {\n> > > +\n> > > +LOG_DECLARE_CATEGORY(IPAProxy)\n> > > +\n> > > +class IPAProxyThread : public IPAProxy, public Object\n> > > +{\n> > > +public:\n> > > +\tIPAProxyThread(IPAModule *ipam);\n> > > +\n> > > +\tint init() override;\n> > > +\tint start() override;\n> > > +\tvoid stop() override;\n> > > +\n> > > +\tvoid configure(const std::map &streamConfig,\n> > > +\t\t const std::map &entityControls) override;\n> > > +\tvoid mapBuffers(const std::vector &buffers) override;\n> > > +\tvoid unmapBuffers(const std::vector &ids) override;\n> > > +\tvoid processEvent(const IPAOperationData &event) override;\n> > > +\n> > > +private:\n> > > +\tvoid queueFrameAction(unsigned int frame, const IPAOperationData &data);\n> > > +\n> > > +\t/* Helper class to invoke processEvent() in another thread. */\n> > > +\tclass ThreadProxy : public Object\n> > > +\t{\n> > > +\tpublic:\n> > > +\t\tvoid setIPA(IPAInterface *ipa)\n> > > +\t\t{\n> > > +\t\t\tipa_ = ipa;\n> > > +\t\t}\n> > > +\n> > > +\t\tvoid stop()\n> > > +\t\t{\n> > > +\t\t\tipa_->stop();\n> > > +\t\t}\n> > > +\n> > > +\t\tvoid processEvent(const IPAOperationData &event)\n> > > +\t\t{\n> > > +\t\t\tipa_->processEvent(event);\n> > > +\t\t}\n> > > +\n> > > +\tprivate:\n> > > +\t\tIPAInterface *ipa_;\n> > > +\t};\n> > > +\n> > > +\tThread thread_;\n> > > +\tThreadProxy proxy_;\n> > > +\tstd::unique_ptr ipa_;\n> > > +};\n> > > +\n> > > +IPAProxyThread::IPAProxyThread(IPAModule *ipam)\n> > > +{\n> > > +\tif (!ipam->load())\n> > > +\t\treturn;\n> > > +\n> > > +\tstruct ipa_context *ctx = ipam->createContext();\n> > > +\tif (!ctx) {\n> > > +\t\tLOG(IPAProxy, Error)\n> > > +\t\t\t<< \"Failed to create IPA context for \" << ipam->path();\n> > > +\t\treturn;\n> > > +\t}\n> > > +\n> > > +\tipa_ = std::make_unique(ctx);\n> > > +\tproxy_.setIPA(ipa_.get());\n> > > +\n> > > +\t/*\n> > > +\t * Proxy the queueFrameAction signal to dispatch it in the caller's\n> > > +\t * thread.\n> > > +\t */\n> > > +\tipa_->queueFrameAction.connect(this, &IPAProxyThread::queueFrameAction);\n> > > +\n> > > +\tvalid_ = true;\n> > > +}\n> > > +\n> > > +int IPAProxyThread::init()\n> > > +{\n> > > +\treturn ipa_->init();\n> > > +}\n> > > +\n> > > +int IPAProxyThread::start()\n> > > +{\n> > > +\tthread_.start();\n> > > +\tproxy_.moveToThread(&thread_);\n> > \n> > If we stop and restart, we will move the proxy to the thread twice. It\n> > shouldn't be an issue, but I wonder if we should move the proxy at init\n> > time, or even in the constructor. I *think* that moving and object to a\n> > thread that hasn't been started yet is fine, but please test it :-)\n> \n> I will test it.\n> \n> > > +\n> > > +\treturn ipa_->start();\n> > \n> > Shouldn't start be called in the target thread, as the thread is already\n> > running ?\n> \n> Good point.\n> \n> > > +}\n> > > +\n> > > +void IPAProxyThread::stop()\n> > > +{\n> > > +\tproxy_.invokeMethod(&ThreadProxy::stop, ConnectionTypeBlocking);\n> > > +\n> > > +\tthread_.exit();\n> > > +\tthread_.wait();\n> > > +}\n> > > +\n> > > +void IPAProxyThread::configure(const std::map &streamConfig,\n> > > +\t\t\t const std::map &entityControls)\n> > > +{\n> > > +\tipa_->configure(streamConfig, entityControls);\n> > > +}\n> > > +\n> > > +void IPAProxyThread::mapBuffers(const std::vector &buffers)\n> > > +{\n> > > +\tipa_->mapBuffers(buffers);\n> > > +}\n> > > +\n> > > +void IPAProxyThread::unmapBuffers(const std::vector &ids)\n> > > +{\n> > > +\tipa_->unmapBuffers(ids);\n> > > +}\n> > > +\n> > > +void IPAProxyThread::processEvent(const IPAOperationData &event)\n> > > +{\n> > > +\t/* Dispatch the processEvent() call to the thread. */\n> > > +\tproxy_.invokeMethod(&ThreadProxy::processEvent, ConnectionTypeQueued,\n> > > +\t\t\t event);\n> > > +}\n> > > +\n> > > +void IPAProxyThread::queueFrameAction(unsigned int frame, const IPAOperationData &data)\n> > > +{\n> > \n> > Instead of blocking frame actions in the pipeline handlers, would it\n> > make sense to block them here when we're stopping ? This function is\n> > called in the pipeline handler thread, and so won't race with stop().\n> \n> I think that could be neat. I thought about it but decided against it as \n> it could create discrepancies in behavior for different IPAProxy \n> implementations.\n> \n> But since we will have relative few IPAProxy implementations and \n> hopefully many pipelines I think it could make sens to move it here, I \n> will do so for next version, if you don't object.\n\nI don't object as I've proposed it :-) Please make sure to update the\nproxy documentation to explain that the queueFrameAction signal will not\nbe delivered after stop() returns.\n\n> > > +\tIPAInterface::queueFrameAction.emit(frame, data);\n> > > +}\n> > > +\n> > > +REGISTER_IPA_PROXY(IPAProxyThread)\n> > > +\n> > > +} /* namespace libcamera */\n> > > diff --git a/src/libcamera/proxy/meson.build b/src/libcamera/proxy/meson.build\n> > > index efc1132302176f63..6c00d5f30ad2e5f0 100644\n> > > --- a/src/libcamera/proxy/meson.build\n> > > +++ b/src/libcamera/proxy/meson.build\n> > > @@ -1,3 +1,4 @@\n> > > libcamera_sources += files([\n> > > 'ipa_proxy_linux.cpp',\n> > > + 'ipa_proxy_thread.cpp',\n> > > ])","headers":{"Return-Path":"","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 3CD3460412\n\tfor ;\n\tThu, 26 Mar 2020 22:53:10 +0100 (CET)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id B863D2DC;\n\tThu, 26 Mar 2020 22:53:09 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"AGsn2i/Y\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1585259589;\n\tbh=kD1SaNhKKhX3N46bNE5KbkWEnTAep5nLmNBnFAecbNE=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=AGsn2i/YOOiXzP3ImqA9BHe8HtOvTSvC3rlc8ffo/oXSSxmiycT6aKFJeCWeElzHN\n\tpcK167Q2pmqemZ56Jj+uLud2aMv/Kqe8N9H+X5qwm3mUh2h6FhGwiy/b1DTcKk1GKC\n\tUha6aZtIsV+7LSa6rMNOo/X6XonJ1KvOGwfzluOc=","Date":"Thu, 26 Mar 2020 23:53:05 +0200","From":"Laurent Pinchart ","To":"Niklas =?utf-8?q?S=C3=B6derlund?= ","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200326215305.GG20581@pendragon.ideasonboard.com>","References":"<20200326160819.4088361-1-niklas.soderlund@ragnatech.se>\n\t<20200326160819.4088361-8-niklas.soderlund@ragnatech.se>\n\t<20200326213726.GE20581@pendragon.ideasonboard.com>\n\t<20200326214843.GB1844461@oden.dyn.berto.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200326214843.GB1844461@oden.dyn.berto.se>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [RFCv2 7/7] libcamera: ipa_manager: Proxy\n\topen-source IPAs to a thread","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","X-List-Received-Date":"Thu, 26 Mar 2020 21:53:10 -0000"}}]