[{"id":26057,"web_url":"https://patchwork.libcamera.org/comment/26057/","msgid":"<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>","date":"2022-12-13T15:55:07","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":3,"url":"https://patchwork.libcamera.org/api/people/3/","name":"Jacopo Mondi","email":"jacopo@jmondi.org"},"content":"Hi Umang,\n\nOn Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> This patch introduces a new constraint in the pipeline handler base\n> class which allows various derived pipeline handlers to set and\n> restrict the number of maximum in-flight requests that can be queued to\n> the underlying components (for e.g. the IPA).\n\nIt's my understanding that the max number of in-flight requests is\nwhat the properties::PipelineDepth property expresses.\n\nCan't we access that property at queue request time, or maybe better we\ncould cache it at camera registration time, since it's constant ?\n\nAlso, can this feature be tested in test/ ? Probably not, as it\nrequires a compliant pipeline handler ? Maybe vimc can be instrumented\n?\n\n>\n> The pipeline handler is now equipped with the responsibility of not to\n> over queue the requests to the underlying layers. The derived\n> pipeline handler (or even IPA) can also have various kind of requests\n> queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> the application queues the requests at a rate where these kind of\n> queues can over-flow. The patch ensures that the base PipelineHandler\n> will never let the requests overflow to underlying layers, once the\n> derived pipeline handler sets its maximum capacity of handling\n> in-flight requests using PipelineHandler::setMaxQueueRequests().\n>\n> The queue request management introduced in the pipeline handler base\n> class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> pipeline handlers.\n>\n> Signed-off-by: Umang Jain \n> ---\n> include/libcamera/internal/pipeline_handler.h | 4 ++\n> src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> 4 files changed, 56 insertions(+), 1 deletion(-)\n>\n> diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> index ec4f662d..83f8bd9f 100644\n> --- a/include/libcamera/internal/pipeline_handler.h\n> +++ b/include/libcamera/internal/pipeline_handler.h\n> @@ -70,6 +70,7 @@ public:\n> protected:\n> \tvoid registerCamera(std::shared_ptr camera);\n> \tvoid hotplugMediaDevice(MediaDevice *media);\n> +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n>\n> \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> \tvirtual void stopDevice(Camera *camera) = 0;\n> @@ -97,6 +98,9 @@ private:\n> \tMutex lock_;\n> \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n>\n> +\tuint32_t maxQueueRequests_;\n> +\tuint32_t requestsQueueCounter_;\n> +\n> \tfriend class PipelineHandlerFactoryBase;\n> };\n>\n> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> index e4d79ea4..d1d42f78 100644\n> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> {\n> +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> }\n>\n> std::unique_ptr\n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index eb9ad65c..a48adba9 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> \t: PipelineHandler(manager), hasSelfPath_(true)\n> {\n> +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> }\n>\n> std::unique_ptr\n> diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> index cfade490..103f9db0 100644\n> --- a/src/libcamera/pipeline_handler.cpp\n> +++ b/src/libcamera/pipeline_handler.cpp\n> @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> * through the PipelineHandlerFactoryBase::create() function.\n> */\n> PipelineHandler::PipelineHandler(CameraManager *manager)\n> -\t: manager_(manager), useCount_(0)\n> +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> +\t requestsQueueCounter_(0)\n> {\n> }\n>\n> @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> \tCamera::Private *data = camera->_d();\n> \tdata->queuedRequests_.push_back(request);\n>\n> +\tif (maxQueueRequests_)\n> +\t\trequestsQueueCounter_++;\n> +\n> \trequest->_d()->sequence_ = data->requestSequence_++;\n>\n> \tif (request->_d()->cancelled_) {\n> @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> \t\tif (!request->_d()->prepared_)\n> \t\t\tbreak;\n>\n> +\t\tif (maxQueueRequests_ &&\n> +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> +\t\t\tbreak;\n> +\n> \t\tdoQueueRequest(request);\n> \t\twaitingRequests_.pop();\n> \t}\n> @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> \t\tASSERT(!req->hasPendingBuffers());\n> \t\tdata->queuedRequests_.pop_front();\n> \t\tcamera->requestComplete(req);\n> +\n> +\t\tif (maxQueueRequests_)\n> +\t\t\trequestsQueueCounter_--;\n> \t}\n> }\n>\n> @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> * constant for the whole lifetime of the pipeline handler.\n> */\n>\n> +/**\n> + * \\var PipelineHandler::maxQueueRequests_\n> + * \\brief Maximum number of in-flight requests that can be queued\n> + *\n> + * A hardware can handle a certain number of maximum requests at a given\n> + * point. If such a constraint exists, set maxQueueRequests_ via\n> + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> + *\n> + * The derived pipeline handler can choose not to define such constraint as\n> + * well. In that case, the derived pipeline handler can avoid setting\n> + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> + * \\a requestsQueueCounter_ will be 0.\n> + */\n> +\n> +/**\n> + * \\var PipelineHandler::requestsQueueCounter_\n> + * \\brief Number of requests queued to the underlying hardware\n> + *\n> + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> + * requestsQueueCounter_ reflects the number of requests queued\n> + * to the underlying hardware by the pipeline handler.\n> + */\n> +\n> +/**\n> + * \\brief Sets the maximum number of requests that can be queued\n> + * \\param[in] maxRequests Maximum number of in-flight requests\n> + *\n> + * A hardware can handle a certain number of requests at a given point.\n> + * This function sets the maximum number of in-flight requests that can\n> + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> + * handler should set the maximum number of in-flight requests it can handle\n> + * at a given point using this function, if at all such a constraint exists.\n> + */\n> +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> +{\n> +\tmaxQueueRequests_ = maxRequests;\n> +}\n> +\n> /**\n> * \\fn PipelineHandler::name()\n> * \\brief Retrieve the pipeline handler name\n> --\n> 2.38.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6E62EBDE6B\n\tfor ;\n\tTue, 13 Dec 2022 15:55:11 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C688E63340;\n\tTue, 13 Dec 2022 16:55:10 +0100 (CET)","from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net\n\t[IPv6:2001:4b98:dc4:8::229])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 0C23D63340\n\tfor ;\n\tTue, 13 Dec 2022 16:55:09 +0100 (CET)","(Authenticated sender: jacopo@jmondi.org)\n\tby mail.gandi.net (Postfix) with ESMTPSA id 7AA9FFF806;\n\tTue, 13 Dec 2022 15:55:08 +0000 (UTC)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1670946910;\n\tbh=Zv7XiJXNJpXw1YLLBCu65J3FtskU/SMSSZKeBOz7jvw=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=juC64DVeyZa4ASraBKASE++7edwfIOgfvVtS1fTii05tFLtWUcBKoT16+G3of1Xpb\n\tzCgGP8zppOJYe9QIuszqXl0i7Tnv1sqmYmTkrR38n+xMhVEWMiAirAE3rmfKRv/rV3\n\t1Pd0WXIRtkuXYe4wzhRT5IUaraZUl6e7rB7p8bTVALolKD5dpg46hd4H26IWRb6Cr9\n\t4xgseZt+3vbfiKulgS1FnXD1IvAfrDCKm6kXt6hK2pifstBiHAh+keHCglcD8afNpH\n\tyUQicGQpBpLUBHS4/pi/GZGA7rNpELO9G8MPwm2DAw8MW188jsZ36TQqThnjHwLwbX\n\tyZ6S/rNZE8bQQ==","Date":"Tue, 13 Dec 2022 16:55:07 +0100","To":"Umang Jain ","Message-ID":"<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20221212150256.69004-3-umang.jain@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26058,"web_url":"https://patchwork.libcamera.org/comment/26058/","msgid":"<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>","date":"2022-12-14T05:37:25","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi Jacopo,\n\nOn 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> Hi Umang,\n>\n> On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n>> This patch introduces a new constraint in the pipeline handler base\n>> class which allows various derived pipeline handlers to set and\n>> restrict the number of maximum in-flight requests that can be queued to\n>> the underlying components (for e.g. the IPA).\n> It's my understanding that the max number of in-flight requests is\n> what the properties::PipelineDepth property expresses.\n\nIndeed, that's why understanding as well. The properties::PipelineDepth \nshould be set by the pipeline-handlers.\n\nWe need to identify and atleast agree upon the correct PipelineDepth for \nthe two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some \ngrey areas there as well\n>\n> Can't we access that property at queue request time, or maybe better we\n> could cache it at camera registration time, since it's constant ?\n\nIf we intent to use the properties::PipelineDepth, as the size of the \nFCQueue - it needs to exist / set before IPA is constructed.\n\n(See IPAIPU3::IPAIPU3() for context)\n\nSo I think queue request time is quite late there. Camera registeration \ntime might be possible with some changes (looking at \nPipelineHandlerIPU3::registerCameras())\n\n\n>\n> Also, can this feature be tested in test/ ? Probably not, as it\n> requires a compliant pipeline handler ? Maybe vimc can be instrumented\n\nI have been thinking about that as well. It's shouldn't be too difficult.\n> ?\n>\n>> The pipeline handler is now equipped with the responsibility of not to\n>> over queue the requests to the underlying layers. The derived\n>> pipeline handler (or even IPA) can also have various kind of requests\n>> queue(arrays, queues or ring-buffer) hence, it might be an issue where\n>> the application queues the requests at a rate where these kind of\n>> queues can over-flow. The patch ensures that the base PipelineHandler\n>> will never let the requests overflow to underlying layers, once the\n>> derived pipeline handler sets its maximum capacity of handling\n>> in-flight requests using PipelineHandler::setMaxQueueRequests().\n>>\n>> The queue request management introduced in the pipeline handler base\n>> class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n>> prevent over-writing of frame contexts (i.e. FCQueue) in these two\n>> pipeline handlers.\n>>\n>> Signed-off-by: Umang Jain \n>> ---\n>> include/libcamera/internal/pipeline_handler.h | 4 ++\n>> src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n>> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n>> src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n>> 4 files changed, 56 insertions(+), 1 deletion(-)\n>>\n>> diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n>> index ec4f662d..83f8bd9f 100644\n>> --- a/include/libcamera/internal/pipeline_handler.h\n>> +++ b/include/libcamera/internal/pipeline_handler.h\n>> @@ -70,6 +70,7 @@ public:\n>> protected:\n>> \tvoid registerCamera(std::shared_ptr camera);\n>> \tvoid hotplugMediaDevice(MediaDevice *media);\n>> +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n>>\n>> \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n>> \tvirtual void stopDevice(Camera *camera) = 0;\n>> @@ -97,6 +98,9 @@ private:\n>> \tMutex lock_;\n>> \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n>>\n>> +\tuint32_t maxQueueRequests_;\n>> +\tuint32_t requestsQueueCounter_;\n>> +\n>> \tfriend class PipelineHandlerFactoryBase;\n>> };\n>>\n>> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n>> index e4d79ea4..d1d42f78 100644\n>> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n>> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n>> @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n>> PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n>> \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n>> {\n>> +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n>> }\n>>\n>> std::unique_ptr\n>> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>> index eb9ad65c..a48adba9 100644\n>> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>> @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n>> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n>> \t: PipelineHandler(manager), hasSelfPath_(true)\n>> {\n>> +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n>> }\n>>\n>> std::unique_ptr\n>> diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n>> index cfade490..103f9db0 100644\n>> --- a/src/libcamera/pipeline_handler.cpp\n>> +++ b/src/libcamera/pipeline_handler.cpp\n>> @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n>> * through the PipelineHandlerFactoryBase::create() function.\n>> */\n>> PipelineHandler::PipelineHandler(CameraManager *manager)\n>> -\t: manager_(manager), useCount_(0)\n>> +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n>> +\t requestsQueueCounter_(0)\n>> {\n>> }\n>>\n>> @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n>> \tCamera::Private *data = camera->_d();\n>> \tdata->queuedRequests_.push_back(request);\n>>\n>> +\tif (maxQueueRequests_)\n>> +\t\trequestsQueueCounter_++;\n>> +\n>> \trequest->_d()->sequence_ = data->requestSequence_++;\n>>\n>> \tif (request->_d()->cancelled_) {\n>> @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n>> \t\tif (!request->_d()->prepared_)\n>> \t\t\tbreak;\n>>\n>> +\t\tif (maxQueueRequests_ &&\n>> +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n>> +\t\t\tbreak;\n>> +\n>> \t\tdoQueueRequest(request);\n>> \t\twaitingRequests_.pop();\n>> \t}\n>> @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n>> \t\tASSERT(!req->hasPendingBuffers());\n>> \t\tdata->queuedRequests_.pop_front();\n>> \t\tcamera->requestComplete(req);\n>> +\n>> +\t\tif (maxQueueRequests_)\n>> +\t\t\trequestsQueueCounter_--;\n>> \t}\n>> }\n>>\n>> @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n>> * constant for the whole lifetime of the pipeline handler.\n>> */\n>>\n>> +/**\n>> + * \\var PipelineHandler::maxQueueRequests_\n>> + * \\brief Maximum number of in-flight requests that can be queued\n>> + *\n>> + * A hardware can handle a certain number of maximum requests at a given\n>> + * point. If such a constraint exists, set maxQueueRequests_ via\n>> + * \\a setMaxQueueRequests() in the derived pipeline handler.\n>> + *\n>> + * The derived pipeline handler can choose not to define such constraint as\n>> + * well. In that case, the derived pipeline handler can avoid setting\n>> + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n>> + * \\a requestsQueueCounter_ will be 0.\n>> + */\n>> +\n>> +/**\n>> + * \\var PipelineHandler::requestsQueueCounter_\n>> + * \\brief Number of requests queued to the underlying hardware\n>> + *\n>> + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n>> + * requestsQueueCounter_ reflects the number of requests queued\n>> + * to the underlying hardware by the pipeline handler.\n>> + */\n>> +\n>> +/**\n>> + * \\brief Sets the maximum number of requests that can be queued\n>> + * \\param[in] maxRequests Maximum number of in-flight requests\n>> + *\n>> + * A hardware can handle a certain number of requests at a given point.\n>> + * This function sets the maximum number of in-flight requests that can\n>> + * be queued to the hardware by the pipeline handler. Each derived pipeline\n>> + * handler should set the maximum number of in-flight requests it can handle\n>> + * at a given point using this function, if at all such a constraint exists.\n>> + */\n>> +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n>> +{\n>> +\tmaxQueueRequests_ = maxRequests;\n>> +}\n>> +\n>> /**\n>> * \\fn PipelineHandler::name()\n>> * \\brief Retrieve the pipeline handler name\n>> --\n>> 2.38.1\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id C0511BDE6B\n\tfor ;\n\tWed, 14 Dec 2022 05:37:35 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 26AD463362;\n\tWed, 14 Dec 2022 06:37:35 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 5653461507\n\tfor ;\n\tWed, 14 Dec 2022 06:37:33 +0100 (CET)","from [IPV6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d] (unknown\n\t[IPv6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id E16EE4A7;\n\tWed, 14 Dec 2022 06:37:30 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1670996255;\n\tbh=hUVMal0oQpwP2GT7Uo4edTjHOwd089Va5atCeNvkM9A=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=HA9WcHZZDb25CWE3bCtX2TjY46XSW7pCbrIsilwk2nTl0+AQY3S4SVEjA8zfzKRGs\n\tpmjOcS9RflB1BwAUirG+SW8eK5H2ZR4hXuFm5OVrm+xJN9ho18R+mqBC5llYpmw67I\n\tCjTFkdOlGweQFyWrGmOh+whqt9vH73MjloRdT5MBZ5AAE3Zq8ewG3q+08W52dncPTe\n\tQQzI83marcIqZ2qGvrcS0EOBgWW9TpkMdG9jL1XKHpYvJVTQ5faqOA26H6955R45A1\n\tr/CHURwfVN0S59edlh+9n7OWmp0dd/ztxATXp80hZ9rXICdUkrUBepp4R7rDrR50/v\n\tCbEwJu5ZaV4wg==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1670996253;\n\tbh=hUVMal0oQpwP2GT7Uo4edTjHOwd089Va5atCeNvkM9A=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=JI9l8RgNheSYRLxJinaucuJ9et+3hsZ7gSmk2jVxOIHdgK41ythmCnqaJhliSvHHI\n\tOIfs5Ny35LEe8Hb1HXkyVn13Lo6EMG9dwelAMrzI+9UBGF8dPTUVHkNWYYvHiWj02B\n\tcmqlUWm4UqC0ycTMmvKC/t6KFE9MRxAibZw1Dv9E="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"JI9l8RgN\"; dkim-atps=neutral","Message-ID":"<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>","Date":"Wed, 14 Dec 2022 11:07:25 +0530","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n\tThunderbird/102.5.1","To":"Jacopo Mondi ","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>","Content-Language":"en-US","In-Reply-To":"<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Umang Jain via libcamera-devel ","Reply-To":"Umang Jain ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26060,"web_url":"https://patchwork.libcamera.org/comment/26060/","msgid":"<20221214075908.xsu4xxl375b47ztk@uno.localdomain>","date":"2022-12-14T07:59:08","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":3,"url":"https://patchwork.libcamera.org/api/people/3/","name":"Jacopo Mondi","email":"jacopo@jmondi.org"},"content":"Hi Umang\n\nOn Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> Hi Jacopo,\n>\n> On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > Hi Umang,\n> >\n> > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > This patch introduces a new constraint in the pipeline handler base\n> > > class which allows various derived pipeline handlers to set and\n> > > restrict the number of maximum in-flight requests that can be queued to\n> > > the underlying components (for e.g. the IPA).\n> > It's my understanding that the max number of in-flight requests is\n> > what the properties::PipelineDepth property expresses.\n>\n> Indeed, that's why understanding as well. The properties::PipelineDepth\n> should be set by the pipeline-handlers.\n>\n> We need to identify and atleast agree upon the correct PipelineDepth for the\n> two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> areas there as well\n\nI think 16 is fine for now. At least doesn't change what we already\nhave today\n\n> >\n> > Can't we access that property at queue request time, or maybe better we\n> > could cache it at camera registration time, since it's constant ?\n>\n> If we intent to use the properties::PipelineDepth, as the size of the\n> FCQueue - it needs to exist / set before IPA is constructed.\n>\n> (See IPAIPU3::IPAIPU3() for context)\n>\n\nThe property can still be defined in the IPA header as IPA can access\nit safely.\n\nPipeline Handlers will have to register the property -before- the\ncamera is registered, but looking at IPU3 that's already the case now\n(data->properties_ gets populated before calling registerCamera())\n\n\n> So I think queue request time is quite late there. Camera registeration time\n> might be possible with some changes (looking at\n> PipelineHandlerIPU3::registerCameras())\n\nWhat I meant is either\n\n1) At PipelineHandler::queueRequest() time, the PipelineHandler base\nclass accesses Camera::properties() and fetches\nproperties::PipelineDepth\n2) At PipelineHandler::registerCamera() time the property value is\ncached, but this would require to cache per-camera data in\nPipelineHandler, something we don't have, or to populate a class\nmember of Camera::Private for that purpose..\n\nSorry for not having been clear\n\n\n>\n>\n> >\n> > Also, can this feature be tested in test/ ? Probably not, as it\n> > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n>\n> I have been thinking about that as well. It's shouldn't be too difficult.\n> > ?\n> >\n> > > The pipeline handler is now equipped with the responsibility of not to\n> > > over queue the requests to the underlying layers. The derived\n> > > pipeline handler (or even IPA) can also have various kind of requests\n> > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > the application queues the requests at a rate where these kind of\n> > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > will never let the requests overflow to underlying layers, once the\n> > > derived pipeline handler sets its maximum capacity of handling\n> > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > >\n> > > The queue request management introduced in the pipeline handler base\n> > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > pipeline handlers.\n> > >\n> > > Signed-off-by: Umang Jain \n> > > ---\n> > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > >\n> > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > index ec4f662d..83f8bd9f 100644\n> > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > @@ -70,6 +70,7 @@ public:\n> > > protected:\n> > > \tvoid registerCamera(std::shared_ptr camera);\n> > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > >\n> > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > @@ -97,6 +98,9 @@ private:\n> > > \tMutex lock_;\n> > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > >\n> > > +\tuint32_t maxQueueRequests_;\n> > > +\tuint32_t requestsQueueCounter_;\n> > > +\n> > > \tfriend class PipelineHandlerFactoryBase;\n> > > };\n> > >\n> > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > index e4d79ea4..d1d42f78 100644\n> > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > {\n> > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > }\n> > >\n> > > std::unique_ptr\n> > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > index eb9ad65c..a48adba9 100644\n> > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > {\n> > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > }\n> > >\n> > > std::unique_ptr\n> > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > index cfade490..103f9db0 100644\n> > > --- a/src/libcamera/pipeline_handler.cpp\n> > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > * through the PipelineHandlerFactoryBase::create() function.\n> > > */\n> > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > -\t: manager_(manager), useCount_(0)\n> > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > +\t requestsQueueCounter_(0)\n> > > {\n> > > }\n> > >\n> > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > \tCamera::Private *data = camera->_d();\n> > > \tdata->queuedRequests_.push_back(request);\n> > >\n> > > +\tif (maxQueueRequests_)\n> > > +\t\trequestsQueueCounter_++;\n> > > +\n> > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > >\n> > > \tif (request->_d()->cancelled_) {\n> > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > \t\tif (!request->_d()->prepared_)\n> > > \t\t\tbreak;\n> > >\n> > > +\t\tif (maxQueueRequests_ &&\n> > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > +\t\t\tbreak;\n> > > +\n> > > \t\tdoQueueRequest(request);\n> > > \t\twaitingRequests_.pop();\n> > > \t}\n> > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > \t\tASSERT(!req->hasPendingBuffers());\n> > > \t\tdata->queuedRequests_.pop_front();\n> > > \t\tcamera->requestComplete(req);\n> > > +\n> > > +\t\tif (maxQueueRequests_)\n> > > +\t\t\trequestsQueueCounter_--;\n> > > \t}\n> > > }\n> > >\n> > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > * constant for the whole lifetime of the pipeline handler.\n> > > */\n> > >\n> > > +/**\n> > > + * \\var PipelineHandler::maxQueueRequests_\n> > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > + *\n> > > + * A hardware can handle a certain number of maximum requests at a given\n> > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > + *\n> > > + * The derived pipeline handler can choose not to define such constraint as\n> > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > + * \\a requestsQueueCounter_ will be 0.\n> > > + */\n> > > +\n> > > +/**\n> > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > + * \\brief Number of requests queued to the underlying hardware\n> > > + *\n> > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > + * to the underlying hardware by the pipeline handler.\n> > > + */\n> > > +\n> > > +/**\n> > > + * \\brief Sets the maximum number of requests that can be queued\n> > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > + *\n> > > + * A hardware can handle a certain number of requests at a given point.\n> > > + * This function sets the maximum number of in-flight requests that can\n> > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > + * handler should set the maximum number of in-flight requests it can handle\n> > > + * at a given point using this function, if at all such a constraint exists.\n> > > + */\n> > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > +{\n> > > +\tmaxQueueRequests_ = maxRequests;\n> > > +}\n> > > +\n> > > /**\n> > > * \\fn PipelineHandler::name()\n> > > * \\brief Retrieve the pipeline handler name\n> > > --\n> > > 2.38.1\n> > >\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 140D1BDE6B\n\tfor ;\n\tWed, 14 Dec 2022 07:59:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6B4AA63354;\n\tWed, 14 Dec 2022 08:59:12 +0100 (CET)","from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net\n\t[217.70.183.199])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 339A161507\n\tfor ;\n\tWed, 14 Dec 2022 08:59:10 +0100 (CET)","(Authenticated sender: jacopo@jmondi.org)\n\tby mail.gandi.net (Postfix) with ESMTPSA id B5898FF804;\n\tWed, 14 Dec 2022 07:59:09 +0000 (UTC)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671004752;\n\tbh=DCrIkT21AveUoP08skVZ3f6zD9oOg60UxVgsTKmMMB4=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=pGAHG3SDnA0Nq+tAIQ6xemZynwl9DqJOkpZfnnJ6hWxFBjQ9HsQ+ApUyuUPhC0Rin\n\tRm/gnkG/NzQjSCGOmdf0NPTIFPX+xGbkfSpO2W1eVOXpToYOyC18iIIX3fVLlErVpK\n\tg0m91Ck+aDNTu+N8x3LgaPgmPg3ZkHGXCnB9va/D1X2D/LVgQ0cuzRBdcu5zQv3ZsW\n\t/3Z9gcsJ/kULk38PuYtTPkNyx6bt8H4fIn8pZWp3iXl3JdztudJa1iX4uQsGXSD0Ai\n\t9IwpdtE+IjW36fXWudV8bfZe2GtGBjcsdceyZUXeieNPYtt0eguI46Z8I04tqoP7at\n\t0DZ+tpFqr5QIw==","Date":"Wed, 14 Dec 2022 08:59:08 +0100","To":"Umang Jain ","Message-ID":"<20221214075908.xsu4xxl375b47ztk@uno.localdomain>","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26061,"web_url":"https://patchwork.libcamera.org/comment/26061/","msgid":"","date":"2022-12-14T08:41:12","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi Jacopo\n\nOn 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> Hi Umang\n>\n> On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n>> Hi Jacopo,\n>>\n>> On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n>>> Hi Umang,\n>>>\n>>> On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n>>>> This patch introduces a new constraint in the pipeline handler base\n>>>> class which allows various derived pipeline handlers to set and\n>>>> restrict the number of maximum in-flight requests that can be queued to\n>>>> the underlying components (for e.g. the IPA).\n>>> It's my understanding that the max number of in-flight requests is\n>>> what the properties::PipelineDepth property expresses.\n>> Indeed, that's why understanding as well. The properties::PipelineDepth\n>> should be set by the pipeline-handlers.\n>>\n>> We need to identify and atleast agree upon the correct PipelineDepth for the\n>> two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n>> areas there as well\n> I think 16 is fine for now. At least doesn't change what we already\n> have today\n>\n>>> Can't we access that property at queue request time, or maybe better we\n>>> could cache it at camera registration time, since it's constant ?\n>> If we intent to use the properties::PipelineDepth, as the size of the\n>> FCQueue - it needs to exist / set before IPA is constructed.\n>>\n>> (See IPAIPU3::IPAIPU3() for context)\n>>\n> The property can still be defined in the IPA header as IPA can access\n> it safely.\n\nYou mean via a global const variable like it's done in Patch 1/2 ?\n>\n> Pipeline Handlers will have to register the property -before- the\n> camera is registered, but looking at IPU3 that's already the case now\n> (data->properties_ gets populated before calling registerCamera())\n>\n>\n>> So I think queue request time is quite late there. Camera registeration time\n>> might be possible with some changes (looking at\n>> PipelineHandlerIPU3::registerCameras())\n> What I meant is either\n>\n> 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> class accesses Camera::properties() and fetches\n> properties::PipelineDepth\n\nI think it's a over-kill here. queueRequest() is a recurring call and I \ndon't want fetching of PipelineDepth here.\n> 2) At PipelineHandler::registerCamera() time the property value is\n> cached, but this would require to cache per-camera data in\n> PipelineHandler, something we don't have, or to populate a class\n> member of Camera::Private for that purpose..\n\nThat's interesting because I initially didn't think of multiple cameras \nregistered with one instance of pipeline-handler.\nIt also makes me wonder -\n\n- Is properties::PipelineDepth Camera specific or platform-specific ? \nSeems now it's Camera specific\n- Then defining it IPA headers won't be a good option, in cases where \nPipelineDepth for different cameras differ.\n\nNeed some thinking ...\n\nBut none the less, this is a something that can be done on top with \nwider discussion / context on PipelineDepth. It shouldn't necessarily \nblock the series.\n>\n> Sorry for not having been clear\n>\n>\n>>\n>>> Also, can this feature be tested in test/ ? Probably not, as it\n>>> requires a compliant pipeline handler ? Maybe vimc can be instrumented\n>> I have been thinking about that as well. It's shouldn't be too difficult.\n>>> ?\n>>>\n>>>> The pipeline handler is now equipped with the responsibility of not to\n>>>> over queue the requests to the underlying layers. The derived\n>>>> pipeline handler (or even IPA) can also have various kind of requests\n>>>> queue(arrays, queues or ring-buffer) hence, it might be an issue where\n>>>> the application queues the requests at a rate where these kind of\n>>>> queues can over-flow. The patch ensures that the base PipelineHandler\n>>>> will never let the requests overflow to underlying layers, once the\n>>>> derived pipeline handler sets its maximum capacity of handling\n>>>> in-flight requests using PipelineHandler::setMaxQueueRequests().\n>>>>\n>>>> The queue request management introduced in the pipeline handler base\n>>>> class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n>>>> prevent over-writing of frame contexts (i.e. FCQueue) in these two\n>>>> pipeline handlers.\n>>>>\n>>>> Signed-off-by: Umang Jain \n>>>> ---\n>>>> include/libcamera/internal/pipeline_handler.h | 4 ++\n>>>> src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n>>>> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n>>>> src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n>>>> 4 files changed, 56 insertions(+), 1 deletion(-)\n>>>>\n>>>> diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n>>>> index ec4f662d..83f8bd9f 100644\n>>>> --- a/include/libcamera/internal/pipeline_handler.h\n>>>> +++ b/include/libcamera/internal/pipeline_handler.h\n>>>> @@ -70,6 +70,7 @@ public:\n>>>> protected:\n>>>> \tvoid registerCamera(std::shared_ptr camera);\n>>>> \tvoid hotplugMediaDevice(MediaDevice *media);\n>>>> +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n>>>>\n>>>> \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n>>>> \tvirtual void stopDevice(Camera *camera) = 0;\n>>>> @@ -97,6 +98,9 @@ private:\n>>>> \tMutex lock_;\n>>>> \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n>>>>\n>>>> +\tuint32_t maxQueueRequests_;\n>>>> +\tuint32_t requestsQueueCounter_;\n>>>> +\n>>>> \tfriend class PipelineHandlerFactoryBase;\n>>>> };\n>>>>\n>>>> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n>>>> index e4d79ea4..d1d42f78 100644\n>>>> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n>>>> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n>>>> @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n>>>> PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n>>>> \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n>>>> {\n>>>> +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n>>>> }\n>>>>\n>>>> std::unique_ptr\n>>>> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>>>> index eb9ad65c..a48adba9 100644\n>>>> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>>>> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n>>>> @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n>>>> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n>>>> \t: PipelineHandler(manager), hasSelfPath_(true)\n>>>> {\n>>>> +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n>>>> }\n>>>>\n>>>> std::unique_ptr\n>>>> diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n>>>> index cfade490..103f9db0 100644\n>>>> --- a/src/libcamera/pipeline_handler.cpp\n>>>> +++ b/src/libcamera/pipeline_handler.cpp\n>>>> @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n>>>> * through the PipelineHandlerFactoryBase::create() function.\n>>>> */\n>>>> PipelineHandler::PipelineHandler(CameraManager *manager)\n>>>> -\t: manager_(manager), useCount_(0)\n>>>> +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n>>>> +\t requestsQueueCounter_(0)\n>>>> {\n>>>> }\n>>>>\n>>>> @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n>>>> \tCamera::Private *data = camera->_d();\n>>>> \tdata->queuedRequests_.push_back(request);\n>>>>\n>>>> +\tif (maxQueueRequests_)\n>>>> +\t\trequestsQueueCounter_++;\n>>>> +\n>>>> \trequest->_d()->sequence_ = data->requestSequence_++;\n>>>>\n>>>> \tif (request->_d()->cancelled_) {\n>>>> @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n>>>> \t\tif (!request->_d()->prepared_)\n>>>> \t\t\tbreak;\n>>>>\n>>>> +\t\tif (maxQueueRequests_ &&\n>>>> +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n>>>> +\t\t\tbreak;\n>>>> +\n>>>> \t\tdoQueueRequest(request);\n>>>> \t\twaitingRequests_.pop();\n>>>> \t}\n>>>> @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n>>>> \t\tASSERT(!req->hasPendingBuffers());\n>>>> \t\tdata->queuedRequests_.pop_front();\n>>>> \t\tcamera->requestComplete(req);\n>>>> +\n>>>> +\t\tif (maxQueueRequests_)\n>>>> +\t\t\trequestsQueueCounter_--;\n>>>> \t}\n>>>> }\n>>>>\n>>>> @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n>>>> * constant for the whole lifetime of the pipeline handler.\n>>>> */\n>>>>\n>>>> +/**\n>>>> + * \\var PipelineHandler::maxQueueRequests_\n>>>> + * \\brief Maximum number of in-flight requests that can be queued\n>>>> + *\n>>>> + * A hardware can handle a certain number of maximum requests at a given\n>>>> + * point. If such a constraint exists, set maxQueueRequests_ via\n>>>> + * \\a setMaxQueueRequests() in the derived pipeline handler.\n>>>> + *\n>>>> + * The derived pipeline handler can choose not to define such constraint as\n>>>> + * well. In that case, the derived pipeline handler can avoid setting\n>>>> + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n>>>> + * \\a requestsQueueCounter_ will be 0.\n>>>> + */\n>>>> +\n>>>> +/**\n>>>> + * \\var PipelineHandler::requestsQueueCounter_\n>>>> + * \\brief Number of requests queued to the underlying hardware\n>>>> + *\n>>>> + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n>>>> + * requestsQueueCounter_ reflects the number of requests queued\n>>>> + * to the underlying hardware by the pipeline handler.\n>>>> + */\n>>>> +\n>>>> +/**\n>>>> + * \\brief Sets the maximum number of requests that can be queued\n>>>> + * \\param[in] maxRequests Maximum number of in-flight requests\n>>>> + *\n>>>> + * A hardware can handle a certain number of requests at a given point.\n>>>> + * This function sets the maximum number of in-flight requests that can\n>>>> + * be queued to the hardware by the pipeline handler. Each derived pipeline\n>>>> + * handler should set the maximum number of in-flight requests it can handle\n>>>> + * at a given point using this function, if at all such a constraint exists.\n>>>> + */\n>>>> +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n>>>> +{\n>>>> +\tmaxQueueRequests_ = maxRequests;\n>>>> +}\n>>>> +\n>>>> /**\n>>>> * \\fn PipelineHandler::name()\n>>>> * \\brief Retrieve the pipeline handler name\n>>>> --\n>>>> 2.38.1\n>>>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 9C2F7C31E9\n\tfor ;\n\tWed, 14 Dec 2022 08:41:24 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id E855C63362;\n\tWed, 14 Dec 2022 09:41:23 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BF370603D0\n\tfor ;\n\tWed, 14 Dec 2022 09:41:22 +0100 (CET)","from [IPV6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d] (unknown\n\t[IPv6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 7500B4A7;\n\tWed, 14 Dec 2022 09:41:21 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671007283;\n\tbh=8xy5A+AyJPG1tDxtNS6IwjH097J9H4mOjCmDrxifGlM=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=FOBVFKr2miW5FKe3O98egLEk51chvUax/QgDv4H8vg06EAJ5bslHbexu8/w3rBp7V\n\tg5I9vGj3+s+tN470v8pEYfBLDv+SSPLH0qUywuopLRabS2BAbQ7Bcf8N/8msvBNVV9\n\tMPKqI1p6Eu8nLz4qqbRKIJlxwmPwQZGJayVuOMstDSd4a0u9GpfZS8jA4BP8j1R1Ws\n\tolkGq7Ulj49AMqgZM62whhs2egKtvvMumnCg16ipTOb+aKV/Y7JiCHOfrd87ECDoQv\n\tDhGFPQJPGIVRbh7L2x2OEW3w/gykjC3JX7o0pnklt6EXSBUeVIA85sUQSs30z+DJPm\n\ta7kRU2rhmj5pw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671007282;\n\tbh=8xy5A+AyJPG1tDxtNS6IwjH097J9H4mOjCmDrxifGlM=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=oxHxk8BVD83LV2OkPWV8hvWD414T7lzRIBz0p06wbSxwnr7D2ZzbM+AcvSVJPo1Kv\n\tlBuDgpHMhcTXaoSPeUBdAFdYY7jH4hSKpBxU6av0TGjPmKsOr+eTf64bLMdF7GCzIk\n\tW9O3KcuzRoOXyjbzQ9Dqml5xPNxusa7KkgVj92z8="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"oxHxk8BV\"; dkim-atps=neutral","Message-ID":"","Date":"Wed, 14 Dec 2022 14:11:12 +0530","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n\tThunderbird/102.5.1","Content-Language":"en-US","To":"Jacopo Mondi ","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>","In-Reply-To":"<20221214075908.xsu4xxl375b47ztk@uno.localdomain>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Umang Jain via libcamera-devel ","Reply-To":"Umang Jain ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26064,"web_url":"https://patchwork.libcamera.org/comment/26064/","msgid":"<20221214091954.igijxckzroa5buyl@uno.localdomain>","date":"2022-12-14T09:19:54","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":3,"url":"https://patchwork.libcamera.org/api/people/3/","name":"Jacopo Mondi","email":"jacopo@jmondi.org"},"content":"Hi Umang\n\nOn Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> Hi Jacopo\n>\n> On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > Hi Umang\n> >\n> > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > Hi Jacopo,\n> > >\n> > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > Hi Umang,\n> > > >\n> > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > > > This patch introduces a new constraint in the pipeline handler base\n> > > > > class which allows various derived pipeline handlers to set and\n> > > > > restrict the number of maximum in-flight requests that can be queued to\n> > > > > the underlying components (for e.g. the IPA).\n> > > > It's my understanding that the max number of in-flight requests is\n> > > > what the properties::PipelineDepth property expresses.\n> > > Indeed, that's why understanding as well. The properties::PipelineDepth\n> > > should be set by the pipeline-handlers.\n> > >\n> > > We need to identify and atleast agree upon the correct PipelineDepth for the\n> > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> > > areas there as well\n> > I think 16 is fine for now. At least doesn't change what we already\n> > have today\n> >\n> > > > Can't we access that property at queue request time, or maybe better we\n> > > > could cache it at camera registration time, since it's constant ?\n> > > If we intent to use the properties::PipelineDepth, as the size of the\n> > > FCQueue - it needs to exist / set before IPA is constructed.\n> > >\n> > > (See IPAIPU3::IPAIPU3() for context)\n> > >\n> > The property can still be defined in the IPA header as IPA can access\n> > it safely.\n>\n> You mean via a global const variable like it's done in Patch 1/2 ?\n\nYes, like in patch 1/2\n\n\n> >\n> > Pipeline Handlers will have to register the property -before- the\n> > camera is registered, but looking at IPU3 that's already the case now\n> > (data->properties_ gets populated before calling registerCamera())\n> >\n> >\n> > > So I think queue request time is quite late there. Camera registeration time\n> > > might be possible with some changes (looking at\n> > > PipelineHandlerIPU3::registerCameras())\n> > What I meant is either\n> >\n> > 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> > class accesses Camera::properties() and fetches\n> > properties::PipelineDepth\n>\n> I think it's a over-kill here. queueRequest() is a recurring call and I\n> don't want fetching of PipelineDepth here.\n\nWhile I consider a lookup on an unordered_map<> of a few items not\nthat expensive, I agree it's in an hot path, and it would be better to\ncache the value somewhere. Caching it inside Camera or Camera::Private\nis a duplication, as it's already part of\nCamera::Private::properties_, but I would be fine with the\nPipelineHandler base class doing something like this (as proposed in\nmy previous 2) point below)\n\nvoid PipelineHandler::registerCamera(std::shared_ptr camera)\n{\n\n\t/* Cache the pipeline depth in the Camera private data. */\n const auto &pipelineDepth = camera->properties().get(properties::PipelineDepth);\n\n\tCamera::Private *data = camera->_d();\n data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n\n ...\n}\n\nOtherwise the PipelineHandler base class can be added with a class\nmember, and each registred camera will overwrite that value, it's less\nnice, but all cameras from the same PH instance will have the same\npipeline depth, so...\n\n\n> > 2) At PipelineHandler::registerCamera() time the property value is\n> > cached, but this would require to cache per-camera data in\n> > PipelineHandler, something we don't have, or to populate a class\n> > member of Camera::Private for that purpose..\n>\n> That's interesting because I initially didn't think of multiple cameras\n> registered with one instance of pipeline-handler.\n> It also makes me wonder -\n>\n> - Is properties::PipelineDepth Camera specific or platform-specific ? Seems\n> now it's Camera specific\n> - Then defining it IPA headers won't be a good option, in cases where\n> PipelineDepth for different cameras differ.\n\nWhy camera specific ? Each camera will indeed register it as a\nproperty for applications to retrieve it, but the\nvalue depends on the platform afaict and it can be a compile-time constant\ndefined by the IPA headers. Even better, the same compile-time\nconstant will be used to size the FCQ.\n\n>\n> Need some thinking ...\n>\n> But none the less, this is a something that can be done on top with wider\n> discussion / context on PipelineDepth. It shouldn't necessarily block the\n> series.\n\nMy whole point is that I think we should find a better mechanism for\nregistering pipeline depth compared to having pipeline handlers call\nsetMaxQueueRequests() as it requires an explicit action that can\neasily be overlooked, while we want the queue overflow protection to\nhappen transparently if a pipeline handler register\nproperties::PipelineDepth ? Am I overthinking this ?\n\n\n> >\n> > Sorry for not having been clear\n> >\n> >\n> > >\n> > > > Also, can this feature be tested in test/ ? Probably not, as it\n> > > > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n> > > I have been thinking about that as well. It's shouldn't be too difficult.\n> > > > ?\n> > > >\n> > > > > The pipeline handler is now equipped with the responsibility of not to\n> > > > > over queue the requests to the underlying layers. The derived\n> > > > > pipeline handler (or even IPA) can also have various kind of requests\n> > > > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > > > the application queues the requests at a rate where these kind of\n> > > > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > > > will never let the requests overflow to underlying layers, once the\n> > > > > derived pipeline handler sets its maximum capacity of handling\n> > > > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > > > >\n> > > > > The queue request management introduced in the pipeline handler base\n> > > > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > > > pipeline handlers.\n> > > > >\n> > > > > Signed-off-by: Umang Jain \n> > > > > ---\n> > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > >\n> > > > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > > > index ec4f662d..83f8bd9f 100644\n> > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > @@ -70,6 +70,7 @@ public:\n> > > > > protected:\n> > > > > \tvoid registerCamera(std::shared_ptr camera);\n> > > > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > > > >\n> > > > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > > > @@ -97,6 +98,9 @@ private:\n> > > > > \tMutex lock_;\n> > > > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > >\n> > > > > +\tuint32_t maxQueueRequests_;\n> > > > > +\tuint32_t requestsQueueCounter_;\n> > > > > +\n> > > > > \tfriend class PipelineHandlerFactoryBase;\n> > > > > };\n> > > > >\n> > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > index e4d79ea4..d1d42f78 100644\n> > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > {\n> > > > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > }\n> > > > >\n> > > > > std::unique_ptr\n> > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > index eb9ad65c..a48adba9 100644\n> > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > > > {\n> > > > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > }\n> > > > >\n> > > > > std::unique_ptr\n> > > > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > > > index cfade490..103f9db0 100644\n> > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > * through the PipelineHandlerFactoryBase::create() function.\n> > > > > */\n> > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > -\t: manager_(manager), useCount_(0)\n> > > > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > > > +\t requestsQueueCounter_(0)\n> > > > > {\n> > > > > }\n> > > > >\n> > > > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > > > \tCamera::Private *data = camera->_d();\n> > > > > \tdata->queuedRequests_.push_back(request);\n> > > > >\n> > > > > +\tif (maxQueueRequests_)\n> > > > > +\t\trequestsQueueCounter_++;\n> > > > > +\n> > > > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > > > >\n> > > > > \tif (request->_d()->cancelled_) {\n> > > > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > > > \t\tif (!request->_d()->prepared_)\n> > > > > \t\t\tbreak;\n> > > > >\n> > > > > +\t\tif (maxQueueRequests_ &&\n> > > > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > > > +\t\t\tbreak;\n> > > > > +\n> > > > > \t\tdoQueueRequest(request);\n> > > > > \t\twaitingRequests_.pop();\n> > > > > \t}\n> > > > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > > > \t\tASSERT(!req->hasPendingBuffers());\n> > > > > \t\tdata->queuedRequests_.pop_front();\n> > > > > \t\tcamera->requestComplete(req);\n> > > > > +\n> > > > > +\t\tif (maxQueueRequests_)\n> > > > > +\t\t\trequestsQueueCounter_--;\n> > > > > \t}\n> > > > > }\n> > > > >\n> > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > * constant for the whole lifetime of the pipeline handler.\n> > > > > */\n> > > > >\n> > > > > +/**\n> > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > > > + *\n> > > > > + * A hardware can handle a certain number of maximum requests at a given\n> > > > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > > > + *\n> > > > > + * The derived pipeline handler can choose not to define such constraint as\n> > > > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > + */\n> > > > > +\n> > > > > +/**\n> > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > + * \\brief Number of requests queued to the underlying hardware\n> > > > > + *\n> > > > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > + */\n> > > > > +\n> > > > > +/**\n> > > > > + * \\brief Sets the maximum number of requests that can be queued\n> > > > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > > > + *\n> > > > > + * A hardware can handle a certain number of requests at a given point.\n> > > > > + * This function sets the maximum number of in-flight requests that can\n> > > > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > > > + * handler should set the maximum number of in-flight requests it can handle\n> > > > > + * at a given point using this function, if at all such a constraint exists.\n> > > > > + */\n> > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > > > +{\n> > > > > +\tmaxQueueRequests_ = maxRequests;\n> > > > > +}\n> > > > > +\n> > > > > /**\n> > > > > * \\fn PipelineHandler::name()\n> > > > > * \\brief Retrieve the pipeline handler name\n> > > > > --\n> > > > > 2.38.1\n> > > > >\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4C053C31E9\n\tfor ;\n\tWed, 14 Dec 2022 09:19:58 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A1C4363363;\n\tWed, 14 Dec 2022 10:19:57 +0100 (CET)","from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net\n\t[217.70.183.198])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 8D66F603D0\n\tfor ;\n\tWed, 14 Dec 2022 10:19:56 +0100 (CET)","(Authenticated sender: jacopo@jmondi.org)\n\tby mail.gandi.net (Postfix) with ESMTPSA id 011D9C0008;\n\tWed, 14 Dec 2022 09:19:55 +0000 (UTC)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671009597;\n\tbh=uCSOaTK6D4KE+TvgXut2JbaEs1JTgbuy/g44RLsVWUQ=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=DE0DFzfsrvWF5oI6eSvAdLwzKT6bW0wn1koJcweLlp/jBJueQiocM8i9E3vAp/VSi\n\tifRMro4glImmhB7EpzYl/EnGjm6d5aGE66b/+gQuFThzBytO/eSSik4I8841ete2yB\n\tZzlRivpuvjzzcFT3epmIv+nFH3ZIXvFilU7kf4vqObSGeW5z93nsuStyl3rWuogFpb\n\tGb/LOq7976CTNCIa9hZOYKW0QMCjkGylSqCwMXoM/Z9HhE4p8xQJ3/MtB82B9k+1+9\n\tBolICsfZDr5alq9ynSfAt6TicCR/w61xSvvttWJMQUB1EQ/qZZEv51rJb8jr789b4t\n\tyUxrAIfOScIrg==","Date":"Wed, 14 Dec 2022 10:19:54 +0100","To":"Umang Jain ","Message-ID":"<20221214091954.igijxckzroa5buyl@uno.localdomain>","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26074,"web_url":"https://patchwork.libcamera.org/comment/26074/","msgid":"","date":"2022-12-14T12:13:06","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"Hi,\n\nOn Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via libcamera-devel wrote:\n> Hi Umang\n> \n> On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > Hi Jacopo\n> >\n> > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > Hi Umang\n> > >\n> > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > Hi Jacopo,\n> > > >\n> > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > Hi Umang,\n> > > > >\n> > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > > > > This patch introduces a new constraint in the pipeline handler base\n> > > > > > class which allows various derived pipeline handlers to set and\n> > > > > > restrict the number of maximum in-flight requests that can be queued to\n> > > > > > the underlying components (for e.g. the IPA).\n> > > > > It's my understanding that the max number of in-flight requests is\n> > > > > what the properties::PipelineDepth property expresses.\n> > > > Indeed, that's why understanding as well. The properties::PipelineDepth\n> > > > should be set by the pipeline-handlers.\n\n(Besides the fact that I can't find properties::PipelineDepth)\ncontrols::PipelineDepth says that it \"Specifies the number of pipeline\nstages the frame went through from when it was exposed to when the final\ncompleted result was available to the framework.\", which to me sounds\nlike the *minimum* amount of in-flight requests.\n\nI thought that here we're defining the *maximum* allowed in-flight\nrequests. Not that all of the requests have to actually be doing\nsomething in hardware, but that it's the amount of requests the the\npipeline handler is capable of handling internally without overflowing.\n\n> > > >\n> > > > We need to identify and atleast agree upon the correct PipelineDepth for the\n> > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> > > > areas there as well\n> > > I think 16 is fine for now. At least doesn't change what we already\n> > > have today\n> > >\n> > > > > Can't we access that property at queue request time, or maybe better we\n> > > > > could cache it at camera registration time, since it's constant ?\n> > > > If we intent to use the properties::PipelineDepth, as the size of the\n> > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > >\n> > > > (See IPAIPU3::IPAIPU3() for context)\n> > > >\n> > > The property can still be defined in the IPA header as IPA can access\n> > > it safely.\n> >\n> > You mean via a global const variable like it's done in Patch 1/2 ?\n> \n> Yes, like in patch 1/2\n> \n> \n> > >\n> > > Pipeline Handlers will have to register the property -before- the\n> > > camera is registered, but looking at IPU3 that's already the case now\n> > > (data->properties_ gets populated before calling registerCamera())\n> > >\n> > >\n> > > > So I think queue request time is quite late there. Camera registeration time\n> > > > might be possible with some changes (looking at\n> > > > PipelineHandlerIPU3::registerCameras())\n> > > What I meant is either\n> > >\n> > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> > > class accesses Camera::properties() and fetches\n> > > properties::PipelineDepth\n> >\n> > I think it's a over-kill here. queueRequest() is a recurring call and I\n> > don't want fetching of PipelineDepth here.\n\nOh one hand, yeah imo getting it from Camera::properties() is more\ncorrect, but on the other hand it does indeed seem overkill.\n\n> \n> While I consider a lookup on an unordered_map<> of a few items not\n> that expensive, I agree it's in an hot path, and it would be better to\n> cache the value somewhere. Caching it inside Camera or Camera::Private\n> is a duplication, as it's already part of\n> Camera::Private::properties_, but I would be fine with the\n> PipelineHandler base class doing something like this (as proposed in\n> my previous 2) point below)\n> \n> void PipelineHandler::registerCamera(std::shared_ptr camera)\n> {\n> \n> \t/* Cache the pipeline depth in the Camera private data. */\n> const auto &pipelineDepth = camera->properties().get(properties::PipelineDepth);\n> \n> \tCamera::Private *data = camera->_d();\n> data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> \n> ...\n> }\n> \n> Otherwise the PipelineHandler base class can be added with a class\n> member, and each registred camera will overwrite that value, it's less\n> nice, but all cameras from the same PH instance will have the same\n> pipeline depth, so...\n\nPerhaps caching would be a good middle ground.\n\n> \n> \n> > > 2) At PipelineHandler::registerCamera() time the property value is\n> > > cached, but this would require to cache per-camera data in\n> > > PipelineHandler, something we don't have, or to populate a class\n> > > member of Camera::Private for that purpose..\n> >\n> > That's interesting because I initially didn't think of multiple cameras\n> > registered with one instance of pipeline-handler.\n> > It also makes me wonder -\n> >\n> > - Is properties::PipelineDepth Camera specific or platform-specific ? Seems\n> > now it's Camera specific\n> > - Then defining it IPA headers won't be a good option, in cases where\n> > PipelineDepth for different cameras differ.\n> \n> Why camera specific ? Each camera will indeed register it as a\n> property for applications to retrieve it, but the\n> value depends on the platform afaict and it can be a compile-time constant\n> defined by the IPA headers. Even better, the same compile-time\n> constant will be used to size the FCQ.\n\nIf it's platform-specific then wouldn't the pipeline handler have to\nreport it based on what platform it's running on? So we can't really\nhardcode a constant. Maybe putting it in properties is indeed better.\n\n> \n> >\n> > Need some thinking ...\n> >\n> > But none the less, this is a something that can be done on top with wider\n> > discussion / context on PipelineDepth. It shouldn't necessarily block the\n> > series.\n> \n> My whole point is that I think we should find a better mechanism for\n> registering pipeline depth compared to having pipeline handlers call\n> setMaxQueueRequests() as it requires an explicit action that can\n> easily be overlooked, while we want the queue overflow protection to\n\nI mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\nand then force *all* pipeline handlers to explictly set it, even if to\nzero. Then it becomes really easy to catch.\n\nOr note it down as a compliancy thing and have lc-compliance test for\nit. I presume this will become a required property to be exposed anyway.\n\nIn any case, the goal is to report to the user the \"maximum number of\nrequests that are allowed to be queued in the camera\", right? So it'll\nhave to go through properties anyway. Perhaps then it's best to expose\nit via properties (as it's already the interface we have for exposing\nproperties to the user), and then the base PipelineHandler can cache it\nfor the queuing guard. That would get rid of the special explicit call,\nand would still achieve the goal.\n\nI suppose then we'd actually need a compliance test for the property,\nbut I guess we have a few more of those anyway.\n\nMy two cents.\n\n\nPaul\n\n> happen transparently if a pipeline handler register\n> properties::PipelineDepth ? Am I overthinking this ?\n> \n> \n> > >\n> > > Sorry for not having been clear\n> > >\n> > >\n> > > >\n> > > > > Also, can this feature be tested in test/ ? Probably not, as it\n> > > > > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n> > > > I have been thinking about that as well. It's shouldn't be too difficult.\n> > > > > ?\n> > > > >\n> > > > > > The pipeline handler is now equipped with the responsibility of not to\n> > > > > > over queue the requests to the underlying layers. The derived\n> > > > > > pipeline handler (or even IPA) can also have various kind of requests\n> > > > > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > > > > the application queues the requests at a rate where these kind of\n> > > > > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > > > > will never let the requests overflow to underlying layers, once the\n> > > > > > derived pipeline handler sets its maximum capacity of handling\n> > > > > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > > > > >\n> > > > > > The queue request management introduced in the pipeline handler base\n> > > > > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > > > > pipeline handlers.\n> > > > > >\n> > > > > > Signed-off-by: Umang Jain \n> > > > > > ---\n> > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > >\n> > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > protected:\n> > > > > > \tvoid registerCamera(std::shared_ptr camera);\n> > > > > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > > > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > > > > >\n> > > > > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > > > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > \tMutex lock_;\n> > > > > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > >\n> > > > > > +\tuint32_t maxQueueRequests_;\n> > > > > > +\tuint32_t requestsQueueCounter_;\n> > > > > > +\n> > > > > > \tfriend class PipelineHandlerFactoryBase;\n> > > > > > };\n> > > > > >\n> > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > > > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > {\n> > > > > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > }\n> > > > > >\n> > > > > > std::unique_ptr\n> > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > > > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > {\n> > > > > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > }\n> > > > > >\n> > > > > > std::unique_ptr\n> > > > > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > > > > index cfade490..103f9db0 100644\n> > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > * through the PipelineHandlerFactoryBase::create() function.\n> > > > > > */\n> > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > -\t: manager_(manager), useCount_(0)\n> > > > > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > > > > +\t requestsQueueCounter_(0)\n> > > > > > {\n> > > > > > }\n> > > > > >\n> > > > > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > > > > \tCamera::Private *data = camera->_d();\n> > > > > > \tdata->queuedRequests_.push_back(request);\n> > > > > >\n> > > > > > +\tif (maxQueueRequests_)\n> > > > > > +\t\trequestsQueueCounter_++;\n> > > > > > +\n> > > > > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > > > > >\n> > > > > > \tif (request->_d()->cancelled_) {\n> > > > > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > > > > \t\tif (!request->_d()->prepared_)\n> > > > > > \t\t\tbreak;\n> > > > > >\n> > > > > > +\t\tif (maxQueueRequests_ &&\n> > > > > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > > > > +\t\t\tbreak;\n> > > > > > +\n> > > > > > \t\tdoQueueRequest(request);\n> > > > > > \t\twaitingRequests_.pop();\n> > > > > > \t}\n> > > > > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > > > > \t\tASSERT(!req->hasPendingBuffers());\n> > > > > > \t\tdata->queuedRequests_.pop_front();\n> > > > > > \t\tcamera->requestComplete(req);\n> > > > > > +\n> > > > > > +\t\tif (maxQueueRequests_)\n> > > > > > +\t\t\trequestsQueueCounter_--;\n> > > > > > \t}\n> > > > > > }\n> > > > > >\n> > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > * constant for the whole lifetime of the pipeline handler.\n> > > > > > */\n> > > > > >\n> > > > > > +/**\n> > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > > > > + *\n> > > > > > + * A hardware can handle a certain number of maximum requests at a given\n> > > > > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > > > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > > > > + *\n> > > > > > + * The derived pipeline handler can choose not to define such constraint as\n> > > > > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > + */\n> > > > > > +\n> > > > > > +/**\n> > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > + * \\brief Number of requests queued to the underlying hardware\n> > > > > > + *\n> > > > > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > > > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > + */\n> > > > > > +\n> > > > > > +/**\n> > > > > > + * \\brief Sets the maximum number of requests that can be queued\n> > > > > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > > > > + *\n> > > > > > + * A hardware can handle a certain number of requests at a given point.\n> > > > > > + * This function sets the maximum number of in-flight requests that can\n> > > > > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > > > > + * handler should set the maximum number of in-flight requests it can handle\n> > > > > > + * at a given point using this function, if at all such a constraint exists.\n> > > > > > + */\n> > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > > > > +{\n> > > > > > +\tmaxQueueRequests_ = maxRequests;\n> > > > > > +}\n> > > > > > +\n> > > > > > /**\n> > > > > > * \\fn PipelineHandler::name()\n> > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > --\n> > > > > > 2.38.1\n> > > > > >\n> >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 7D132C31E9\n\tfor ;\n\tWed, 14 Dec 2022 12:13:17 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1A50663354;\n\tWed, 14 Dec 2022 13:13:17 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D837A61F23\n\tfor ;\n\tWed, 14 Dec 2022 13:13:15 +0100 (CET)","from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp\n\t[175.177.42.159])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 27B1D49C;\n\tWed, 14 Dec 2022 13:13:13 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671019997;\n\tbh=SnnkVIqLmXbA6PLhrfnFLmz++BLMWzNhZ6h1D4C8jwo=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=W07KKjvgZbmpb6350jiW2cahX/ZQMQ+sRflSsaH33rJFjtDGcx8f9QJ6EYEa7ANB6\n\tlECp7wIcDew0V1DcpspL0MpQkY7bF5XuEiA6B8tUcDVyIl/9BYBrstwrOqTJzrQ4nD\n\tb/iSuOCKJVAiALhZSJJmoNT1p/jJpIvo3bhNOM04fvEFEivohRZu1X0OV93vTo0HR6\n\tWqCyPV4g8bCQgHWv7LwlRkEjOPbYGmzNvxNjdA1ZG8zBoHwO47g1KSHWV399ZYuNq6\n\trfpG1Bdh6crir5YHNVMo7X9qTKyW7j2xErBu9+YyID+PHaXARsnJ63/lygD7u/91+L\n\tn9fOZ91H2X2eA==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671019995;\n\tbh=SnnkVIqLmXbA6PLhrfnFLmz++BLMWzNhZ6h1D4C8jwo=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=I17lRhF/MxvfOcSGB2sANPiKGyrYNlJCWfbbH/MzPwjE810YQRZM+UE38R9BZVrDX\n\tJdlDFHIGLFqx86WtC6l7AZSDOIlN9NBicBkZZaS8VsbTPlTjlH2J0xnt4wV5qAzC3O\n\tKGpQ7IwSGpSrxhrvoo/ssy0lgh0daz05AVqvCLQs="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"I17lRhF/\"; dkim-atps=neutral","Date":"Wed, 14 Dec 2022 21:13:06 +0900","To":"Jacopo Mondi ","Message-ID":"","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>","MIME-Version":"1.0","Content-Type":"text/plain; charset=us-ascii","Content-Disposition":"inline","In-Reply-To":"<20221214091954.igijxckzroa5buyl@uno.localdomain>","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Paul Elder via libcamera-devel ","Reply-To":"Paul Elder ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26075,"web_url":"https://patchwork.libcamera.org/comment/26075/","msgid":"<20221214134130.flgk2ztujuwctnei@uno.localdomain>","date":"2022-12-14T13:41:30","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":3,"url":"https://patchwork.libcamera.org/api/people/3/","name":"Jacopo Mondi","email":"jacopo@jmondi.org"},"content":"Hi Paul\n\nOn Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> Hi,\n>\n> On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via libcamera-devel wrote:\n> > Hi Umang\n> >\n> > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > Hi Jacopo\n> > >\n> > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > Hi Umang\n> > > >\n> > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > Hi Jacopo,\n> > > > >\n> > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > Hi Umang,\n> > > > > >\n> > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > > > > > This patch introduces a new constraint in the pipeline handler base\n> > > > > > > class which allows various derived pipeline handlers to set and\n> > > > > > > restrict the number of maximum in-flight requests that can be queued to\n> > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > It's my understanding that the max number of in-flight requests is\n> > > > > > what the properties::PipelineDepth property expresses.\n> > > > > Indeed, that's why understanding as well. The properties::PipelineDepth\n> > > > > should be set by the pipeline-handlers.\n>\n> (Besides the fact that I can't find properties::PipelineDepth)\n> controls::PipelineDepth says that it \"Specifies the number of pipeline\n> stages the frame went through from when it was exposed to when the final\n> completed result was available to the framework.\", which to me sounds\n> like the *minimum* amount of in-flight requests.\n\nIt's a draft control copied in to please Android.\nMy understanding was that it had to be made a non-mutable property\n\n>\n> I thought that here we're defining the *maximum* allowed in-flight\n> requests. Not that all of the requests have to actually be doing\n> something in hardware, but that it's the amount of requests the the\n> pipeline handler is capable of handling internally without overflowing.\n>\n> > > > >\n> > > > > We need to identify and atleast agree upon the correct PipelineDepth for the\n> > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> > > > > areas there as well\n> > > > I think 16 is fine for now. At least doesn't change what we already\n> > > > have today\n> > > >\n> > > > > > Can't we access that property at queue request time, or maybe better we\n> > > > > > could cache it at camera registration time, since it's constant ?\n> > > > > If we intent to use the properties::PipelineDepth, as the size of the\n> > > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > > >\n> > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > >\n> > > > The property can still be defined in the IPA header as IPA can access\n> > > > it safely.\n> > >\n> > > You mean via a global const variable like it's done in Patch 1/2 ?\n> >\n> > Yes, like in patch 1/2\n> >\n> >\n> > > >\n> > > > Pipeline Handlers will have to register the property -before- the\n> > > > camera is registered, but looking at IPU3 that's already the case now\n> > > > (data->properties_ gets populated before calling registerCamera())\n> > > >\n> > > >\n> > > > > So I think queue request time is quite late there. Camera registeration time\n> > > > > might be possible with some changes (looking at\n> > > > > PipelineHandlerIPU3::registerCameras())\n> > > > What I meant is either\n> > > >\n> > > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> > > > class accesses Camera::properties() and fetches\n> > > > properties::PipelineDepth\n> > >\n> > > I think it's a over-kill here. queueRequest() is a recurring call and I\n> > > don't want fetching of PipelineDepth here.\n>\n> Oh one hand, yeah imo getting it from Camera::properties() is more\n> correct, but on the other hand it does indeed seem overkill.\n>\n> >\n> > While I consider a lookup on an unordered_map<> of a few items not\n> > that expensive, I agree it's in an hot path, and it would be better to\n> > cache the value somewhere. Caching it inside Camera or Camera::Private\n> > is a duplication, as it's already part of\n> > Camera::Private::properties_, but I would be fine with the\n> > PipelineHandler base class doing something like this (as proposed in\n> > my previous 2) point below)\n> >\n> > void PipelineHandler::registerCamera(std::shared_ptr camera)\n> > {\n> >\n> > \t/* Cache the pipeline depth in the Camera private data. */\n> > const auto &pipelineDepth = camera->properties().get(properties::PipelineDepth);\n> >\n> > \tCamera::Private *data = camera->_d();\n> > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> >\n> > ...\n> > }\n> >\n> > Otherwise the PipelineHandler base class can be added with a class\n> > member, and each registred camera will overwrite that value, it's less\n> > nice, but all cameras from the same PH instance will have the same\n> > pipeline depth, so...\n>\n> Perhaps caching would be a good middle ground.\n>\n> >\n> >\n> > > > 2) At PipelineHandler::registerCamera() time the property value is\n> > > > cached, but this would require to cache per-camera data in\n> > > > PipelineHandler, something we don't have, or to populate a class\n> > > > member of Camera::Private for that purpose..\n> > >\n> > > That's interesting because I initially didn't think of multiple cameras\n> > > registered with one instance of pipeline-handler.\n> > > It also makes me wonder -\n> > >\n> > > - Is properties::PipelineDepth Camera specific or platform-specific ? Seems\n> > > now it's Camera specific\n> > > - Then defining it IPA headers won't be a good option, in cases where\n> > > PipelineDepth for different cameras differ.\n> >\n> > Why camera specific ? Each camera will indeed register it as a\n> > property for applications to retrieve it, but the\n> > value depends on the platform afaict and it can be a compile-time constant\n> > defined by the IPA headers. Even better, the same compile-time\n> > constant will be used to size the FCQ.\n>\n> If it's platform-specific then wouldn't the pipeline handler have to\n> report it based on what platform it's running on? So we can't really\n>hardcode a constant. Maybe putting it in properties is indeed better.\n\nAs far as I get it, this property applies to pipelines with an ISP (no\nuvc, no simple) and those pipelines by definition runs on a single\nplatform, don't they ?\n\n>\n> >\n> > >\n> > > Need some thinking ...\n> > >\n> > > But none the less, this is a something that can be done on top with wider\n> > > discussion / context on PipelineDepth. It shouldn't necessarily block the\n> > > series.\n> >\n> > My whole point is that I think we should find a better mechanism for\n> > registering pipeline depth compared to having pipeline handlers call\n> > setMaxQueueRequests() as it requires an explicit action that can\n> > easily be overlooked, while we want the queue overflow protection to\n>\n> I mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\n> and then force *all* pipeline handlers to explictly set it, even if to\n> zero. Then it becomes really easy to catch.\n\nThe property doesn't make much sense for UVC, right ?\n\nNot all pipelines will use libipa and have an FCQ (RPi in example at\nthe moment). I don't think forcing all of them is actually necessary.\n\nBut the pipelines that do need to throttle the number of requests in\nflight with the help of the PipelineHandler base class should ideally\nget it for free by simply registering the property ?\n\n>\n> Or note it down as a compliancy thing and have lc-compliance test for\n> it. I presume this will become a required property to be exposed anyway.\n>\n> In any case, the goal is to report to the user the \"maximum number of\n> requests that are allowed to be queued in the camera\", right? So it'll\n> have to go through properties anyway. Perhaps then it's best to expose\n> it via properties (as it's already the interface we have for exposing\n> properties to the user), and then the base PipelineHandler can cache it\n> for the queuing guard. That would get rid of the special explicit call,\n> and would still achieve the goal.\n\nThat would be my preference. There might be a discussions if\nproperties::PipelineDepth is the right property to report it or it\nwill conflict with Android's pipeline depth, which is different.\n\n>\n> I suppose then we'd actually need a compliance test for the property,\n> but I guess we have a few more of those anyway.\n>\n> My two cents.\n\nThanks ;)\n j\n\n>\n>\n> Paul\n>\n> > happen transparently if a pipeline handler register\n> > properties::PipelineDepth ? Am I overthinking this ?\n> >\n> >\n> > > >\n> > > > Sorry for not having been clear\n> > > >\n> > > >\n> > > > >\n> > > > > > Also, can this feature be tested in test/ ? Probably not, as it\n> > > > > > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n> > > > > I have been thinking about that as well. It's shouldn't be too difficult.\n> > > > > > ?\n> > > > > >\n> > > > > > > The pipeline handler is now equipped with the responsibility of not to\n> > > > > > > over queue the requests to the underlying layers. The derived\n> > > > > > > pipeline handler (or even IPA) can also have various kind of requests\n> > > > > > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > > > > > the application queues the requests at a rate where these kind of\n> > > > > > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > > > > > will never let the requests overflow to underlying layers, once the\n> > > > > > > derived pipeline handler sets its maximum capacity of handling\n> > > > > > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > > > > > >\n> > > > > > > The queue request management introduced in the pipeline handler base\n> > > > > > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > > > > > pipeline handlers.\n> > > > > > >\n> > > > > > > Signed-off-by: Umang Jain \n> > > > > > > ---\n> > > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > >\n> > > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > protected:\n> > > > > > > \tvoid registerCamera(std::shared_ptr camera);\n> > > > > > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > > > > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > >\n> > > > > > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > > > > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > \tMutex lock_;\n> > > > > > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > >\n> > > > > > > +\tuint32_t maxQueueRequests_;\n> > > > > > > +\tuint32_t requestsQueueCounter_;\n> > > > > > > +\n> > > > > > > \tfriend class PipelineHandlerFactoryBase;\n> > > > > > > };\n> > > > > > >\n> > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > > > > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > {\n> > > > > > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > }\n> > > > > > >\n> > > > > > > std::unique_ptr\n> > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > > > > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > {\n> > > > > > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > }\n> > > > > > >\n> > > > > > > std::unique_ptr\n> > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > > > > > index cfade490..103f9db0 100644\n> > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > * through the PipelineHandlerFactoryBase::create() function.\n> > > > > > > */\n> > > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > > -\t: manager_(manager), useCount_(0)\n> > > > > > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > > > > > +\t requestsQueueCounter_(0)\n> > > > > > > {\n> > > > > > > }\n> > > > > > >\n> > > > > > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > \tCamera::Private *data = camera->_d();\n> > > > > > > \tdata->queuedRequests_.push_back(request);\n> > > > > > >\n> > > > > > > +\tif (maxQueueRequests_)\n> > > > > > > +\t\trequestsQueueCounter_++;\n> > > > > > > +\n> > > > > > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > > > > > >\n> > > > > > > \tif (request->_d()->cancelled_) {\n> > > > > > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > > > > > \t\tif (!request->_d()->prepared_)\n> > > > > > > \t\t\tbreak;\n> > > > > > >\n> > > > > > > +\t\tif (maxQueueRequests_ &&\n> > > > > > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > > > > > +\t\t\tbreak;\n> > > > > > > +\n> > > > > > > \t\tdoQueueRequest(request);\n> > > > > > > \t\twaitingRequests_.pop();\n> > > > > > > \t}\n> > > > > > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > > > > > \t\tASSERT(!req->hasPendingBuffers());\n> > > > > > > \t\tdata->queuedRequests_.pop_front();\n> > > > > > > \t\tcamera->requestComplete(req);\n> > > > > > > +\n> > > > > > > +\t\tif (maxQueueRequests_)\n> > > > > > > +\t\t\trequestsQueueCounter_--;\n> > > > > > > \t}\n> > > > > > > }\n> > > > > > >\n> > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > * constant for the whole lifetime of the pipeline handler.\n> > > > > > > */\n> > > > > > >\n> > > > > > > +/**\n> > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > > > > > + *\n> > > > > > > + * A hardware can handle a certain number of maximum requests at a given\n> > > > > > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > > > > > + *\n> > > > > > > + * The derived pipeline handler can choose not to define such constraint as\n> > > > > > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > + */\n> > > > > > > +\n> > > > > > > +/**\n> > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > + * \\brief Number of requests queued to the underlying hardware\n> > > > > > > + *\n> > > > > > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > > > > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > + */\n> > > > > > > +\n> > > > > > > +/**\n> > > > > > > + * \\brief Sets the maximum number of requests that can be queued\n> > > > > > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > > > > > + *\n> > > > > > > + * A hardware can handle a certain number of requests at a given point.\n> > > > > > > + * This function sets the maximum number of in-flight requests that can\n> > > > > > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > > > > > + * handler should set the maximum number of in-flight requests it can handle\n> > > > > > > + * at a given point using this function, if at all such a constraint exists.\n> > > > > > > + */\n> > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > > > > > +{\n> > > > > > > +\tmaxQueueRequests_ = maxRequests;\n> > > > > > > +}\n> > > > > > > +\n> > > > > > > /**\n> > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > --\n> > > > > > > 2.38.1\n> > > > > > >\n> > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 80C48C3200\n\tfor ;\n\tWed, 14 Dec 2022 13:41:34 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id D979063362;\n\tWed, 14 Dec 2022 14:41:33 +0100 (CET)","from relay3-d.mail.gandi.net (relay3-d.mail.gandi.net\n\t[IPv6:2001:4b98:dc4:8::223])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D444361F23\n\tfor ;\n\tWed, 14 Dec 2022 14:41:32 +0100 (CET)","(Authenticated sender: jacopo@jmondi.org)\n\tby mail.gandi.net (Postfix) with ESMTPSA id 17FB96000B;\n\tWed, 14 Dec 2022 13:41:31 +0000 (UTC)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671025293;\n\tbh=gpzAapWNFrUgf0umQVm8+iqBkAOPvKbA7Lf7Qbiewv4=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=1DYtkPSJ3M3t23yppbztiyuSYfkET2YCMIgZ/QJTxjCEaVIWU8HWZjv48mHEMKfQa\n\t96H7GOlJRIYhlbzRHItRhCp5If4yXFECsZb8ohZbInGC8+PmIEdw7XVTe5nFHUj3Yi\n\trSTPlyJipXRnDP+JKzQbaHN+ilq1coZH3BpZmz4+d6CfQY755ZeKSrFypZfxt8V5b+\n\tPmO00R+YTQcb+ps+e88kH0QLZrv9bzXj7nSVDlnQlZUiZhaW8KPPc12U5X+lXwXu/+\n\tYyItDCBLqnXwHsNeDBGULVMf0dSKxPJz7XXdnuvK9/t4m0udH6jxMY/CxeWnpZNXGO\n\t049lvAMqLbUvQ==","Date":"Wed, 14 Dec 2022 14:41:30 +0100","To":"Paul Elder ","Message-ID":"<20221214134130.flgk2ztujuwctnei@uno.localdomain>","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26082,"web_url":"https://patchwork.libcamera.org/comment/26082/","msgid":"","date":"2022-12-15T05:18:22","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"Hi Jacopo,\n\nOn Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> Hi Paul\n> \n> On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > Hi,\n> >\n> > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via libcamera-devel wrote:\n> > > Hi Umang\n> > >\n> > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > Hi Jacopo\n> > > >\n> > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > Hi Umang\n> > > > >\n> > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > Hi Jacopo,\n> > > > > >\n> > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > Hi Umang,\n> > > > > > >\n> > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > > > > > > This patch introduces a new constraint in the pipeline handler base\n> > > > > > > > class which allows various derived pipeline handlers to set and\n> > > > > > > > restrict the number of maximum in-flight requests that can be queued to\n> > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > It's my understanding that the max number of in-flight requests is\n> > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > Indeed, that's why understanding as well. The properties::PipelineDepth\n> > > > > > should be set by the pipeline-handlers.\n> >\n> > (Besides the fact that I can't find properties::PipelineDepth)\n> > controls::PipelineDepth says that it \"Specifies the number of pipeline\n> > stages the frame went through from when it was exposed to when the final\n> > completed result was available to the framework.\", which to me sounds\n> > like the *minimum* amount of in-flight requests.\n> \n> It's a draft control copied in to please Android.\n> My understanding was that it had to be made a non-mutable property\n\nAh, indeed it says that :)\n\n> \n> >\n> > I thought that here we're defining the *maximum* allowed in-flight\n> > requests. Not that all of the requests have to actually be doing\n> > something in hardware, but that it's the amount of requests the the\n> > pipeline handler is capable of handling internally without overflowing.\n> >\n> > > > > >\n> > > > > > We need to identify and atleast agree upon the correct PipelineDepth for the\n> > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> > > > > > areas there as well\n> > > > > I think 16 is fine for now. At least doesn't change what we already\n> > > > > have today\n> > > > >\n> > > > > > > Can't we access that property at queue request time, or maybe better we\n> > > > > > > could cache it at camera registration time, since it's constant ?\n> > > > > > If we intent to use the properties::PipelineDepth, as the size of the\n> > > > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > > > >\n> > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > >\n> > > > > The property can still be defined in the IPA header as IPA can access\n> > > > > it safely.\n> > > >\n> > > > You mean via a global const variable like it's done in Patch 1/2 ?\n> > >\n> > > Yes, like in patch 1/2\n> > >\n> > >\n> > > > >\n> > > > > Pipeline Handlers will have to register the property -before- the\n> > > > > camera is registered, but looking at IPU3 that's already the case now\n> > > > > (data->properties_ gets populated before calling registerCamera())\n> > > > >\n> > > > >\n> > > > > > So I think queue request time is quite late there. Camera registeration time\n> > > > > > might be possible with some changes (looking at\n> > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > What I meant is either\n> > > > >\n> > > > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> > > > > class accesses Camera::properties() and fetches\n> > > > > properties::PipelineDepth\n> > > >\n> > > > I think it's a over-kill here. queueRequest() is a recurring call and I\n> > > > don't want fetching of PipelineDepth here.\n> >\n> > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > correct, but on the other hand it does indeed seem overkill.\n> >\n> > >\n> > > While I consider a lookup on an unordered_map<> of a few items not\n> > > that expensive, I agree it's in an hot path, and it would be better to\n> > > cache the value somewhere. Caching it inside Camera or Camera::Private\n> > > is a duplication, as it's already part of\n> > > Camera::Private::properties_, but I would be fine with the\n> > > PipelineHandler base class doing something like this (as proposed in\n> > > my previous 2) point below)\n> > >\n> > > void PipelineHandler::registerCamera(std::shared_ptr camera)\n> > > {\n> > >\n> > > \t/* Cache the pipeline depth in the Camera private data. */\n> > > const auto &pipelineDepth = camera->properties().get(properties::PipelineDepth);\n> > >\n> > > \tCamera::Private *data = camera->_d();\n> > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > >\n> > > ...\n> > > }\n> > >\n> > > Otherwise the PipelineHandler base class can be added with a class\n> > > member, and each registred camera will overwrite that value, it's less\n> > > nice, but all cameras from the same PH instance will have the same\n> > > pipeline depth, so...\n> >\n> > Perhaps caching would be a good middle ground.\n> >\n> > >\n> > >\n> > > > > 2) At PipelineHandler::registerCamera() time the property value is\n> > > > > cached, but this would require to cache per-camera data in\n> > > > > PipelineHandler, something we don't have, or to populate a class\n> > > > > member of Camera::Private for that purpose..\n> > > >\n> > > > That's interesting because I initially didn't think of multiple cameras\n> > > > registered with one instance of pipeline-handler.\n> > > > It also makes me wonder -\n> > > >\n> > > > - Is properties::PipelineDepth Camera specific or platform-specific ? Seems\n> > > > now it's Camera specific\n> > > > - Then defining it IPA headers won't be a good option, in cases where\n> > > > PipelineDepth for different cameras differ.\n> > >\n> > > Why camera specific ? Each camera will indeed register it as a\n> > > property for applications to retrieve it, but the\n> > > value depends on the platform afaict and it can be a compile-time constant\n> > > defined by the IPA headers. Even better, the same compile-time\n> > > constant will be used to size the FCQ.\n> >\n> > If it's platform-specific then wouldn't the pipeline handler have to\n> > report it based on what platform it's running on? So we can't really\n> >hardcode a constant. Maybe putting it in properties is indeed better.\n> \n> As far as I get it, this property applies to pipelines with an ISP (no\n> uvc, no simple) and those pipelines by definition runs on a single\n> platform, don't they ?\n\nWell, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\nbut out-of-tree kernel patches)\n\n> \n> >\n> > >\n> > > >\n> > > > Need some thinking ...\n> > > >\n> > > > But none the less, this is a something that can be done on top with wider\n> > > > discussion / context on PipelineDepth. It shouldn't necessarily block the\n> > > > series.\n> > >\n> > > My whole point is that I think we should find a better mechanism for\n> > > registering pipeline depth compared to having pipeline handlers call\n> > > setMaxQueueRequests() as it requires an explicit action that can\n> > > easily be overlooked, while we want the queue overflow protection to\n> >\n> > I mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\n> > and then force *all* pipeline handlers to explictly set it, even if to\n> > zero. Then it becomes really easy to catch.\n> \n> The property doesn't make much sense for UVC, right ?\n> \n> Not all pipelines will use libipa and have an FCQ (RPi in example at\n> the moment). I don't think forcing all of them is actually necessary.\n\nHm, that's true. No initial invalid value then.\n\n> \n> But the pipelines that do need to throttle the number of requests in\n> flight with the help of the PipelineHandler base class should ideally\n> get it for free by simply registering the property ?\n\nYeah that does sound more nice.\n\n> \n> >\n> > Or note it down as a compliancy thing and have lc-compliance test for\n> > it. I presume this will become a required property to be exposed anyway.\n> >\n> > In any case, the goal is to report to the user the \"maximum number of\n> > requests that are allowed to be queued in the camera\", right? So it'll\n> > have to go through properties anyway. Perhaps then it's best to expose\n> > it via properties (as it's already the interface we have for exposing\n> > properties to the user), and then the base PipelineHandler can cache it\n> > for the queuing guard. That would get rid of the special explicit call,\n> > and would still achieve the goal.\n> \n> That would be my preference. There might be a discussions if\n> properties::PipelineDepth is the right property to report it or it\n> will conflict with Android's pipeline depth, which is different.\n\n+1\n\nAlthough to me this sounds separate from PipelineDepth, because my\nunderstanding is that that is the maximum amount of Requests that can be\nin-flight, while what we want here is that plus some extra that the\napplication is allowed to \"overqueue\" that the pipeline handler will\nautomagically handle internally.\n\nAm I misunderstanding something?\n\n\nPaul\n\n> \n> >\n> > I suppose then we'd actually need a compliance test for the property,\n> > but I guess we have a few more of those anyway.\n> >\n> > My two cents.\n> \n> Thanks ;)\n> j\n> \n> >\n> >\n> > Paul\n> >\n> > > happen transparently if a pipeline handler register\n> > > properties::PipelineDepth ? Am I overthinking this ?\n> > >\n> > >\n> > > > >\n> > > > > Sorry for not having been clear\n> > > > >\n> > > > >\n> > > > > >\n> > > > > > > Also, can this feature be tested in test/ ? Probably not, as it\n> > > > > > > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n> > > > > > I have been thinking about that as well. It's shouldn't be too difficult.\n> > > > > > > ?\n> > > > > > >\n> > > > > > > > The pipeline handler is now equipped with the responsibility of not to\n> > > > > > > > over queue the requests to the underlying layers. The derived\n> > > > > > > > pipeline handler (or even IPA) can also have various kind of requests\n> > > > > > > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > > > > > > the application queues the requests at a rate where these kind of\n> > > > > > > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > > > > > > will never let the requests overflow to underlying layers, once the\n> > > > > > > > derived pipeline handler sets its maximum capacity of handling\n> > > > > > > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > > > > > > >\n> > > > > > > > The queue request management introduced in the pipeline handler base\n> > > > > > > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > > > > > > pipeline handlers.\n> > > > > > > >\n> > > > > > > > Signed-off-by: Umang Jain \n> > > > > > > > ---\n> > > > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > > >\n> > > > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > > protected:\n> > > > > > > > \tvoid registerCamera(std::shared_ptr camera);\n> > > > > > > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > > > > > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > > >\n> > > > > > > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > > > > > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > > \tMutex lock_;\n> > > > > > > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > > >\n> > > > > > > > +\tuint32_t maxQueueRequests_;\n> > > > > > > > +\tuint32_t requestsQueueCounter_;\n> > > > > > > > +\n> > > > > > > > \tfriend class PipelineHandlerFactoryBase;\n> > > > > > > > };\n> > > > > > > >\n> > > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > > > > > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > > {\n> > > > > > > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > > }\n> > > > > > > >\n> > > > > > > > std::unique_ptr\n> > > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > > > > > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > > {\n> > > > > > > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > > }\n> > > > > > > >\n> > > > > > > > std::unique_ptr\n> > > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > index cfade490..103f9db0 100644\n> > > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > > * through the PipelineHandlerFactoryBase::create() function.\n> > > > > > > > */\n> > > > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > > > -\t: manager_(manager), useCount_(0)\n> > > > > > > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > > > > > > +\t requestsQueueCounter_(0)\n> > > > > > > > {\n> > > > > > > > }\n> > > > > > > >\n> > > > > > > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > > \tCamera::Private *data = camera->_d();\n> > > > > > > > \tdata->queuedRequests_.push_back(request);\n> > > > > > > >\n> > > > > > > > +\tif (maxQueueRequests_)\n> > > > > > > > +\t\trequestsQueueCounter_++;\n> > > > > > > > +\n> > > > > > > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > > > > > > >\n> > > > > > > > \tif (request->_d()->cancelled_) {\n> > > > > > > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > > > > > > \t\tif (!request->_d()->prepared_)\n> > > > > > > > \t\t\tbreak;\n> > > > > > > >\n> > > > > > > > +\t\tif (maxQueueRequests_ &&\n> > > > > > > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > > > > > > +\t\t\tbreak;\n> > > > > > > > +\n> > > > > > > > \t\tdoQueueRequest(request);\n> > > > > > > > \t\twaitingRequests_.pop();\n> > > > > > > > \t}\n> > > > > > > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > > > > > > \t\tASSERT(!req->hasPendingBuffers());\n> > > > > > > > \t\tdata->queuedRequests_.pop_front();\n> > > > > > > > \t\tcamera->requestComplete(req);\n> > > > > > > > +\n> > > > > > > > +\t\tif (maxQueueRequests_)\n> > > > > > > > +\t\t\trequestsQueueCounter_--;\n> > > > > > > > \t}\n> > > > > > > > }\n> > > > > > > >\n> > > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > > * constant for the whole lifetime of the pipeline handler.\n> > > > > > > > */\n> > > > > > > >\n> > > > > > > > +/**\n> > > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > > > > > > + *\n> > > > > > > > + * A hardware can handle a certain number of maximum requests at a given\n> > > > > > > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > > > > > > + *\n> > > > > > > > + * The derived pipeline handler can choose not to define such constraint as\n> > > > > > > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > > + */\n> > > > > > > > +\n> > > > > > > > +/**\n> > > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > > + * \\brief Number of requests queued to the underlying hardware\n> > > > > > > > + *\n> > > > > > > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > > > > > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > > + */\n> > > > > > > > +\n> > > > > > > > +/**\n> > > > > > > > + * \\brief Sets the maximum number of requests that can be queued\n> > > > > > > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > > > > > > + *\n> > > > > > > > + * A hardware can handle a certain number of requests at a given point.\n> > > > > > > > + * This function sets the maximum number of in-flight requests that can\n> > > > > > > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > > > > > > + * handler should set the maximum number of in-flight requests it can handle\n> > > > > > > > + * at a given point using this function, if at all such a constraint exists.\n> > > > > > > > + */\n> > > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > > > > > > +{\n> > > > > > > > +\tmaxQueueRequests_ = maxRequests;\n> > > > > > > > +}\n> > > > > > > > +\n> > > > > > > > /**\n> > > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > > --\n> > > > > > > > 2.38.1\n> > > > > > > >\n> > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 60C93C31E9\n\tfor ;\n\tThu, 15 Dec 2022 05:18:33 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AC2DF6339D;\n\tThu, 15 Dec 2022 06:18:32 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4E760603CA\n\tfor ;\n\tThu, 15 Dec 2022 06:18:30 +0100 (CET)","from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp\n\t[175.177.42.159])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 7572A4D5;\n\tThu, 15 Dec 2022 06:18:28 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671081512;\n\tbh=oXaiDOCAFjkUZkKWkUC7HaS5k93hxSYtWkiDeSbpC/k=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=KSNnLOJ1r1Nbhfiki2I9/GsRym9qyDYrEISAnQAAWMLAtxQDJJVsJKD8jrMf3QYQq\n\tLLCXuAk37XopOF7F9Rk+uC/aQw5rpaYQ7tSkRTsAF+7JTDP1yYNiGkJOJ4sWfEuV1A\n\tloNSet8h5IiLs43gWoum+rnS+Pt6tuY0evz+TspLgGInEFAZ1F7KPkg/R2rXx0R0MO\n\tzvKzO9+c8gfmG0l4zBswSNUBtl1yAZ+8BX6NY+W1cdkjuTffOcmWuACNQVJ789dx8J\n\tgFw7G7yiH21vNgKx65DLLjFbuLHUayACyp6/BNBj2wDDj6KbbPciHmxbMjY74IF6V4\n\tcUAVJ+TF001vA==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671081509;\n\tbh=oXaiDOCAFjkUZkKWkUC7HaS5k93hxSYtWkiDeSbpC/k=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=JFL5rlBhMMU/xBG01fiOmv/FukluM7zMapVuKPym9q+SIV0N6UVJwTh+9U++PoCoo\n\tzchCErcMdvp+C8KazBBgbBqq0qT4CVUFeKYwTxVy5Zqynj4UeG0a8W6hIWkVIbAoNy\n\tniemsudLBt55oYGTT4AcXxyWHYBDDxzIJUbtv9bQ="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"JFL5rlBh\"; dkim-atps=neutral","Date":"Thu, 15 Dec 2022 14:18:22 +0900","To":"Jacopo Mondi ","Message-ID":"","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>","MIME-Version":"1.0","Content-Type":"text/plain; charset=us-ascii","Content-Disposition":"inline","In-Reply-To":"<20221214134130.flgk2ztujuwctnei@uno.localdomain>","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Paul Elder via libcamera-devel ","Reply-To":"Paul Elder ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26083,"web_url":"https://patchwork.libcamera.org/comment/26083/","msgid":"<20221215085245.iw6teosifzqcgfef@uno.localdomain>","date":"2022-12-15T08:55:40","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":3,"url":"https://patchwork.libcamera.org/api/people/3/","name":"Jacopo Mondi","email":"jacopo@jmondi.org"},"content":"Hi Paul\n\nOn Thu, Dec 15, 2022 at 02:18:22PM +0900, Paul Elder wrote:\n> Hi Jacopo,\n>\n> On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> > Hi Paul\n> >\n> > On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > > Hi,\n> > >\n> > > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via libcamera-devel wrote:\n> > > > Hi Umang\n> > > >\n> > > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > > Hi Jacopo\n> > > > >\n> > > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > > Hi Umang\n> > > > > >\n> > > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > > Hi Umang,\n> > > > > > > >\n> > > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via libcamera-devel wrote:\n> > > > > > > > > This patch introduces a new constraint in the pipeline handler base\n> > > > > > > > > class which allows various derived pipeline handlers to set and\n> > > > > > > > > restrict the number of maximum in-flight requests that can be queued to\n> > > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > > It's my understanding that the max number of in-flight requests is\n> > > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > > Indeed, that's why understanding as well. The properties::PipelineDepth\n> > > > > > > should be set by the pipeline-handlers.\n> > >\n> > > (Besides the fact that I can't find properties::PipelineDepth)\n> > > controls::PipelineDepth says that it \"Specifies the number of pipeline\n> > > stages the frame went through from when it was exposed to when the final\n> > > completed result was available to the framework.\", which to me sounds\n> > > like the *minimum* amount of in-flight requests.\n> >\n> > It's a draft control copied in to please Android.\n> > My understanding was that it had to be made a non-mutable property\n>\n> Ah, indeed it says that :)\n>\n> >\n> > >\n> > > I thought that here we're defining the *maximum* allowed in-flight\n> > > requests. Not that all of the requests have to actually be doing\n> > > something in hardware, but that it's the amount of requests the the\n> > > pipeline handler is capable of handling internally without overflowing.\n> > >\n> > > > > > >\n> > > > > > > We need to identify and atleast agree upon the correct PipelineDepth for the\n> > > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well. Some grey\n> > > > > > > areas there as well\n> > > > > > I think 16 is fine for now. At least doesn't change what we already\n> > > > > > have today\n> > > > > >\n> > > > > > > > Can't we access that property at queue request time, or maybe better we\n> > > > > > > > could cache it at camera registration time, since it's constant ?\n> > > > > > > If we intent to use the properties::PipelineDepth, as the size of the\n> > > > > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > > > > >\n> > > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > > >\n> > > > > > The property can still be defined in the IPA header as IPA can access\n> > > > > > it safely.\n> > > > >\n> > > > > You mean via a global const variable like it's done in Patch 1/2 ?\n> > > >\n> > > > Yes, like in patch 1/2\n> > > >\n> > > >\n> > > > > >\n> > > > > > Pipeline Handlers will have to register the property -before- the\n> > > > > > camera is registered, but looking at IPU3 that's already the case now\n> > > > > > (data->properties_ gets populated before calling registerCamera())\n> > > > > >\n> > > > > >\n> > > > > > > So I think queue request time is quite late there. Camera registeration time\n> > > > > > > might be possible with some changes (looking at\n> > > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > > What I meant is either\n> > > > > >\n> > > > > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler base\n> > > > > > class accesses Camera::properties() and fetches\n> > > > > > properties::PipelineDepth\n> > > > >\n> > > > > I think it's a over-kill here. queueRequest() is a recurring call and I\n> > > > > don't want fetching of PipelineDepth here.\n> > >\n> > > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > > correct, but on the other hand it does indeed seem overkill.\n> > >\n> > > >\n> > > > While I consider a lookup on an unordered_map<> of a few items not\n> > > > that expensive, I agree it's in an hot path, and it would be better to\n> > > > cache the value somewhere. Caching it inside Camera or Camera::Private\n> > > > is a duplication, as it's already part of\n> > > > Camera::Private::properties_, but I would be fine with the\n> > > > PipelineHandler base class doing something like this (as proposed in\n> > > > my previous 2) point below)\n> > > >\n> > > > void PipelineHandler::registerCamera(std::shared_ptr camera)\n> > > > {\n> > > >\n> > > > \t/* Cache the pipeline depth in the Camera private data. */\n> > > > const auto &pipelineDepth = camera->properties().get(properties::PipelineDepth);\n> > > >\n> > > > \tCamera::Private *data = camera->_d();\n> > > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > > >\n> > > > ...\n> > > > }\n> > > >\n> > > > Otherwise the PipelineHandler base class can be added with a class\n> > > > member, and each registred camera will overwrite that value, it's less\n> > > > nice, but all cameras from the same PH instance will have the same\n> > > > pipeline depth, so...\n> > >\n> > > Perhaps caching would be a good middle ground.\n> > >\n> > > >\n> > > >\n> > > > > > 2) At PipelineHandler::registerCamera() time the property value is\n> > > > > > cached, but this would require to cache per-camera data in\n> > > > > > PipelineHandler, something we don't have, or to populate a class\n> > > > > > member of Camera::Private for that purpose..\n> > > > >\n> > > > > That's interesting because I initially didn't think of multiple cameras\n> > > > > registered with one instance of pipeline-handler.\n> > > > > It also makes me wonder -\n> > > > >\n> > > > > - Is properties::PipelineDepth Camera specific or platform-specific ? Seems\n> > > > > now it's Camera specific\n> > > > > - Then defining it IPA headers won't be a good option, in cases where\n> > > > > PipelineDepth for different cameras differ.\n> > > >\n> > > > Why camera specific ? Each camera will indeed register it as a\n> > > > property for applications to retrieve it, but the\n> > > > value depends on the platform afaict and it can be a compile-time constant\n> > > > defined by the IPA headers. Even better, the same compile-time\n> > > > constant will be used to size the FCQ.\n> > >\n> > > If it's platform-specific then wouldn't the pipeline handler have to\n> > > report it based on what platform it's running on? So we can't really\n> > >hardcode a constant. Maybe putting it in properties is indeed better.\n> >\n> > As far as I get it, this property applies to pipelines with an ISP (no\n> > uvc, no simple) and those pipelines by definition runs on a single\n> > platform, don't they ?\n>\n> Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n> but out-of-tree kernel patches)\n>\n\nCorrect, but it does so as it uses the same ISP, same IPA\nimplementation, same FCQ depth ?\n\n> >\n> > >\n> > > >\n> > > > >\n> > > > > Need some thinking ...\n> > > > >\n> > > > > But none the less, this is a something that can be done on top with wider\n> > > > > discussion / context on PipelineDepth. It shouldn't necessarily block the\n> > > > > series.\n> > > >\n> > > > My whole point is that I think we should find a better mechanism for\n> > > > registering pipeline depth compared to having pipeline handlers call\n> > > > setMaxQueueRequests() as it requires an explicit action that can\n> > > > easily be overlooked, while we want the queue overflow protection to\n> > >\n> > > I mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\n> > > and then force *all* pipeline handlers to explictly set it, even if to\n> > > zero. Then it becomes really easy to catch.\n> >\n> > The property doesn't make much sense for UVC, right ?\n> >\n> > Not all pipelines will use libipa and have an FCQ (RPi in example at\n> > the moment). I don't think forcing all of them is actually necessary.\n>\n> Hm, that's true. No initial invalid value then.\n>\n> >\n> > But the pipelines that do need to throttle the number of requests in\n> > flight with the help of the PipelineHandler base class should ideally\n> > get it for free by simply registering the property ?\n>\n> Yeah that does sound more nice.\n>\n> >\n> > >\n> > > Or note it down as a compliancy thing and have lc-compliance test for\n> > > it. I presume this will become a required property to be exposed anyway.\n> > >\n> > > In any case, the goal is to report to the user the \"maximum number of\n> > > requests that are allowed to be queued in the camera\", right? So it'll\n> > > have to go through properties anyway. Perhaps then it's best to expose\n> > > it via properties (as it's already the interface we have for exposing\n> > > properties to the user), and then the base PipelineHandler can cache it\n> > > for the queuing guard. That would get rid of the special explicit call,\n> > > and would still achieve the goal.\n> >\n> > That would be my preference. There might be a discussions if\n> > properties::PipelineDepth is the right property to report it or it\n> > will conflict with Android's pipeline depth, which is different.\n>\n> +1\n>\n> Although to me this sounds separate from PipelineDepth, because my\n> understanding is that that is the maximum amount of Requests that can be\n> in-flight, while what we want here is that plus some extra that the\n> application is allowed to \"overqueue\" that the pipeline handler will\n> automagically handle internally.\n\nI think you're right here.\n\nActually, with this request blocking protection in place it is not\njust \"some extra\" but application can theoretically queue an infinite\nnumber of requests, but only up to X of them will be processed\nconcurrently. Which makes me wonder what is the use for applications\nof knowing that, if the requests queue length from their point of view is\ninfinite (aka they can queue as much requests as they like).\n\nIt is instead more relevant for applications to be able to identify the\nnumber of processing steps a request has to go through before being\ncompleted, as it represents the latency (in frames I presume) before\nan image gets exposed to the time it is delivered to userspace.\n\nThe Android documentation for 'pipelineMaxDepth' reports:\n\n-------------------------------------------------------------------------------\nA typical minimum value for this is 2 (one stage to expose,one stage\nto readout) from the sensor. The ISP then usually adds its own stages\nto do custom HW processing. Further stages may be added by SW\nprocessing.\n\nDepending on what settings are used (e.g. YUV, JPEG) and what\nprocessing is enabled (e.g. face detection), the actual pipeline depth\n(specified by android.request.pipelineDepth) may be less than the max\npipeline depth.\n\nA pipeline depth of X stages is equivalent to a pipeline latency of X\nframe intervals.\n\nThis value will normally be 8 or less, however, for high speed capture\nsession,the max pipeline depth will be up to 8 x size of high speed\ncapture request list.\n-------------------------------------------------------------------------------\n\nand we probably want something very similar.\n\nOk, scratch my properties::PipelineDepth suggestion, but I still don't\nlike very much the explicit function call. Would it make sense to have\na protected PipelineHandler class member for pipelines to populate ?\n\n>\n> Am I misunderstanding something?\n\nnot at all, I probably was :)\n>\n>\n> Paul\n>\n> >\n> > >\n> > > I suppose then we'd actually need a compliance test for the property,\n> > > but I guess we have a few more of those anyway.\n> > >\n> > > My two cents.\n> >\n> > Thanks ;)\n> > j\n> >\n> > >\n> > >\n> > > Paul\n> > >\n> > > > happen transparently if a pipeline handler register\n> > > > properties::PipelineDepth ? Am I overthinking this ?\n> > > >\n> > > >\n> > > > > >\n> > > > > > Sorry for not having been clear\n> > > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > > Also, can this feature be tested in test/ ? Probably not, as it\n> > > > > > > > requires a compliant pipeline handler ? Maybe vimc can be instrumented\n> > > > > > > I have been thinking about that as well. It's shouldn't be too difficult.\n> > > > > > > > ?\n> > > > > > > >\n> > > > > > > > > The pipeline handler is now equipped with the responsibility of not to\n> > > > > > > > > over queue the requests to the underlying layers. The derived\n> > > > > > > > > pipeline handler (or even IPA) can also have various kind of requests\n> > > > > > > > > queue(arrays, queues or ring-buffer) hence, it might be an issue where\n> > > > > > > > > the application queues the requests at a rate where these kind of\n> > > > > > > > > queues can over-flow. The patch ensures that the base PipelineHandler\n> > > > > > > > > will never let the requests overflow to underlying layers, once the\n> > > > > > > > > derived pipeline handler sets its maximum capacity of handling\n> > > > > > > > > in-flight requests using PipelineHandler::setMaxQueueRequests().\n> > > > > > > > >\n> > > > > > > > > The queue request management introduced in the pipeline handler base\n> > > > > > > > > class is now used by the IPU3 and RkISP1 pipeline handlers. This will\n> > > > > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in these two\n> > > > > > > > > pipeline handlers.\n> > > > > > > > >\n> > > > > > > > > Signed-off-by: Umang Jain \n> > > > > > > > > ---\n> > > > > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > > > src/libcamera/pipeline_handler.cpp | 51 ++++++++++++++++++-\n> > > > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > > > >\n> > > > > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > > > protected:\n> > > > > > > > > \tvoid registerCamera(std::shared_ptr camera);\n> > > > > > > > > \tvoid hotplugMediaDevice(MediaDevice *media);\n> > > > > > > > > +\tvoid setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > > > >\n> > > > > > > > > \tvirtual int queueRequestDevice(Camera *camera, Request *request) = 0;\n> > > > > > > > > \tvirtual void stopDevice(Camera *camera) = 0;\n> > > > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > > > \tMutex lock_;\n> > > > > > > > > \tunsigned int useCount_ LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > > > >\n> > > > > > > > > +\tuint32_t maxQueueRequests_;\n> > > > > > > > > +\tuint32_t requestsQueueCounter_;\n> > > > > > > > > +\n> > > > > > > > > \tfriend class PipelineHandlerFactoryBase;\n> > > > > > > > > };\n> > > > > > > > >\n> > > > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status IPU3CameraConfiguration::validate()\n> > > > > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)\n> > > > > > > > > \t: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > > > {\n> > > > > > > > > +\tsetMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > std::unique_ptr\n> > > > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> > > > > > > > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > > > \t: PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > > > {\n> > > > > > > > > +\tsetMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > std::unique_ptr\n> > > > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > index cfade490..103f9db0 100644\n> > > > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > > > * through the PipelineHandlerFactoryBase::create() function.\n> > > > > > > > > */\n> > > > > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > > > > -\t: manager_(manager), useCount_(0)\n> > > > > > > > > +\t: manager_(manager), useCount_(0), maxQueueRequests_(0),\n> > > > > > > > > +\t requestsQueueCounter_(0)\n> > > > > > > > > {\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > @@ -428,6 +429,9 @@ void PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > > > \tCamera::Private *data = camera->_d();\n> > > > > > > > > \tdata->queuedRequests_.push_back(request);\n> > > > > > > > >\n> > > > > > > > > +\tif (maxQueueRequests_)\n> > > > > > > > > +\t\trequestsQueueCounter_++;\n> > > > > > > > > +\n> > > > > > > > > \trequest->_d()->sequence_ = data->requestSequence_++;\n> > > > > > > > >\n> > > > > > > > > \tif (request->_d()->cancelled_) {\n> > > > > > > > > @@ -455,6 +459,10 @@ void PipelineHandler::doQueueRequests()\n> > > > > > > > > \t\tif (!request->_d()->prepared_)\n> > > > > > > > > \t\t\tbreak;\n> > > > > > > > >\n> > > > > > > > > +\t\tif (maxQueueRequests_ &&\n> > > > > > > > > +\t\t requestsQueueCounter_ >= maxQueueRequests_)\n> > > > > > > > > +\t\t\tbreak;\n> > > > > > > > > +\n> > > > > > > > > \t\tdoQueueRequest(request);\n> > > > > > > > > \t\twaitingRequests_.pop();\n> > > > > > > > > \t}\n> > > > > > > > > @@ -531,6 +539,9 @@ void PipelineHandler::completeRequest(Request *request)\n> > > > > > > > > \t\tASSERT(!req->hasPendingBuffers());\n> > > > > > > > > \t\tdata->queuedRequests_.pop_front();\n> > > > > > > > > \t\tcamera->requestComplete(req);\n> > > > > > > > > +\n> > > > > > > > > +\t\tif (maxQueueRequests_)\n> > > > > > > > > +\t\t\trequestsQueueCounter_--;\n> > > > > > > > > \t}\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > > > * constant for the whole lifetime of the pipeline handler.\n> > > > > > > > > */\n> > > > > > > > >\n> > > > > > > > > +/**\n> > > > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > > > + * \\brief Maximum number of in-flight requests that can be queued\n> > > > > > > > > + *\n> > > > > > > > > + * A hardware can handle a certain number of maximum requests at a given\n> > > > > > > > > + * point. If such a constraint exists, set maxQueueRequests_ via\n> > > > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline handler.\n> > > > > > > > > + *\n> > > > > > > > > + * The derived pipeline handler can choose not to define such constraint as\n> > > > > > > > > + * well. In that case, the derived pipeline handler can avoid setting\n> > > > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_ and\n> > > > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > > > + */\n> > > > > > > > > +\n> > > > > > > > > +/**\n> > > > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > > > + * \\brief Number of requests queued to the underlying hardware\n> > > > > > > > > + *\n> > > > > > > > > + * If \\a setMaxQueueRequests() is set by the derived pipeline handler,\n> > > > > > > > > + * requestsQueueCounter_ reflects the number of requests queued\n> > > > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > > > + */\n> > > > > > > > > +\n> > > > > > > > > +/**\n> > > > > > > > > + * \\brief Sets the maximum number of requests that can be queued\n> > > > > > > > > + * \\param[in] maxRequests Maximum number of in-flight requests\n> > > > > > > > > + *\n> > > > > > > > > + * A hardware can handle a certain number of requests at a given point.\n> > > > > > > > > + * This function sets the maximum number of in-flight requests that can\n> > > > > > > > > + * be queued to the hardware by the pipeline handler. Each derived pipeline\n> > > > > > > > > + * handler should set the maximum number of in-flight requests it can handle\n> > > > > > > > > + * at a given point using this function, if at all such a constraint exists.\n> > > > > > > > > + */\n> > > > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t maxRequests)\n> > > > > > > > > +{\n> > > > > > > > > +\tmaxQueueRequests_ = maxRequests;\n> > > > > > > > > +}\n> > > > > > > > > +\n> > > > > > > > > /**\n> > > > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > > > --\n> > > > > > > > > 2.38.1\n> > > > > > > > >\n> > > > >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 8E289C31E9\n\tfor ;\n\tThu, 15 Dec 2022 08:55:44 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 0B9586339B;\n\tThu, 15 Dec 2022 09:55:44 +0100 (CET)","from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net\n\t[217.70.183.194])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BE5E660410\n\tfor ;\n\tThu, 15 Dec 2022 09:55:41 +0100 (CET)","(Authenticated sender: jacopo@jmondi.org)\n\tby mail.gandi.net (Postfix) with ESMTPSA id 1968F40009;\n\tThu, 15 Dec 2022 08:55:40 +0000 (UTC)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671094544;\n\tbh=0c/8ABlpV0x20Gw62pskf+TloeIvumdGxN+LW/r3qCE=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=Y5sgWn3AwPe5KVW4bWWfmuJ6wBCt7XG5vI8yXlS5ZMjb4AVYzHtE2DTN/gF3LsF7X\n\tOmw2Uz9pDppd0ivk26K0fRCTHiNsg07ce8s/JWX64YqmIO9UIRR9cFPHGthakWKHUF\n\tEGpAjEQNFJJrvuMieYAp64nL508mKXdLC7ME5WfOJCFWWDDF6ZIAb1l3HJOVjM32vu\n\tG9MY8wy0RCX5tQBCbPMYiAM1ZDPQAUt7XFsHGpEltxlm+kSrJTEf1X7xCDH6y44d0W\n\tUREM6CLpH2VihiN1GWqvfbnWwqrc33IlzIOekjt2SXQyPn3Xq+mL1uvXV49bAofvXO\n\t+L7XguVAvxQQw==","Date":"Thu, 15 Dec 2022 09:55:40 +0100","To":"Paul Elder ","Message-ID":"<20221215085245.iw6teosifzqcgfef@uno.localdomain>","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26084,"web_url":"https://patchwork.libcamera.org/comment/26084/","msgid":"","date":"2022-12-15T09:43:55","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"Hi all,\n\nOn Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\nlibcamera-devel@lists.libcamera.org> wrote:\n\n> Hi Jacopo,\n>\n> On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> > Hi Paul\n> >\n> > On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > > Hi,\n> > >\n> > > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n> libcamera-devel wrote:\n> > > > Hi Umang\n> > > >\n> > > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > > Hi Jacopo\n> > > > >\n> > > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > > Hi Umang\n> > > > > >\n> > > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > > Hi Jacopo,\n> > > > > > >\n> > > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > > Hi Umang,\n> > > > > > > >\n> > > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n> libcamera-devel wrote:\n> > > > > > > > > This patch introduces a new constraint in the pipeline\n> handler base\n> > > > > > > > > class which allows various derived pipeline handlers to\n> set and\n> > > > > > > > > restrict the number of maximum in-flight requests that can\n> be queued to\n> > > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > > It's my understanding that the max number of in-flight\n> requests is\n> > > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > > Indeed, that's why understanding as well. The\n> properties::PipelineDepth\n> > > > > > > should be set by the pipeline-handlers.\n> > >\n> > > (Besides the fact that I can't find properties::PipelineDepth)\n> > > controls::PipelineDepth says that it \"Specifies the number of pipeline\n> > > stages the frame went through from when it was exposed to when the\n> final\n> > > completed result was available to the framework.\", which to me sounds\n> > > like the *minimum* amount of in-flight requests.\n> >\n> > It's a draft control copied in to please Android.\n> > My understanding was that it had to be made a non-mutable property\n>\n> Ah, indeed it says that :)\n>\n> >\n> > >\n> > > I thought that here we're defining the *maximum* allowed in-flight\n> > > requests. Not that all of the requests have to actually be doing\n> > > something in hardware, but that it's the amount of requests the the\n> > > pipeline handler is capable of handling internally without overflowing.\n> > >\n> > > > > > >\n> > > > > > > We need to identify and atleast agree upon the correct\n> PipelineDepth for the\n> > > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well.\n> Some grey\n> > > > > > > areas there as well\n> > > > > > I think 16 is fine for now. At least doesn't change what we\n> already\n> > > > > > have today\n> > > > > >\n> > > > > > > > Can't we access that property at queue request time, or\n> maybe better we\n> > > > > > > > could cache it at camera registration time, since it's\n> constant ?\n> > > > > > > If we intent to use the properties::PipelineDepth, as the size\n> of the\n> > > > > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > > > > >\n> > > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > > >\n> > > > > > The property can still be defined in the IPA header as IPA can\n> access\n> > > > > > it safely.\n> > > > >\n> > > > > You mean via a global const variable like it's done in Patch 1/2 ?\n> > > >\n> > > > Yes, like in patch 1/2\n> > > >\n> > > >\n> > > > > >\n> > > > > > Pipeline Handlers will have to register the property -before- the\n> > > > > > camera is registered, but looking at IPU3 that's already the\n> case now\n> > > > > > (data->properties_ gets populated before calling\n> registerCamera())\n> > > > > >\n> > > > > >\n> > > > > > > So I think queue request time is quite late there. Camera\n> registeration time\n> > > > > > > might be possible with some changes (looking at\n> > > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > > What I meant is either\n> > > > > >\n> > > > > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler\n> base\n> > > > > > class accesses Camera::properties() and fetches\n> > > > > > properties::PipelineDepth\n> > > > >\n> > > > > I think it's a over-kill here. queueRequest() is a recurring call\n> and I\n> > > > > don't want fetching of PipelineDepth here.\n> > >\n> > > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > > correct, but on the other hand it does indeed seem overkill.\n> > >\n> > > >\n> > > > While I consider a lookup on an unordered_map<> of a few items not\n> > > > that expensive, I agree it's in an hot path, and it would be better\n> to\n> > > > cache the value somewhere. Caching it inside Camera or\n> Camera::Private\n> > > > is a duplication, as it's already part of\n> > > > Camera::Private::properties_, but I would be fine with the\n> > > > PipelineHandler base class doing something like this (as proposed in\n> > > > my previous 2) point below)\n> > > >\n> > > > void PipelineHandler::registerCamera(std::shared_ptr camera)\n> > > > {\n> > > >\n> > > > /* Cache the pipeline depth in the Camera private data. */\n> > > > const auto &pipelineDepth =\n> camera->properties().get(properties::PipelineDepth);\n> > > >\n> > > > Camera::Private *data = camera->_d();\n> > > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > > >\n> > > > ...\n> > > > }\n> > > >\n> > > > Otherwise the PipelineHandler base class can be added with a class\n> > > > member, and each registred camera will overwrite that value, it's\n> less\n> > > > nice, but all cameras from the same PH instance will have the same\n> > > > pipeline depth, so...\n> > >\n> > > Perhaps caching would be a good middle ground.\n> > >\n> > > >\n> > > >\n> > > > > > 2) At PipelineHandler::registerCamera() time the property value\n> is\n> > > > > > cached, but this would require to cache per-camera data in\n> > > > > > PipelineHandler, something we don't have, or to populate a class\n> > > > > > member of Camera::Private for that purpose..\n> > > > >\n> > > > > That's interesting because I initially didn't think of multiple\n> cameras\n> > > > > registered with one instance of pipeline-handler.\n> > > > > It also makes me wonder -\n> > > > >\n> > > > > - Is properties::PipelineDepth Camera specific or\n> platform-specific ? Seems\n> > > > > now it's Camera specific\n> > > > > - Then defining it IPA headers won't be a good option, in cases\n> where\n> > > > > PipelineDepth for different cameras differ.\n> > > >\n> > > > Why camera specific ? Each camera will indeed register it as a\n> > > > property for applications to retrieve it, but the\n> > > > value depends on the platform afaict and it can be a compile-time\n> constant\n> > > > defined by the IPA headers. Even better, the same compile-time\n> > > > constant will be used to size the FCQ.\n> > >\n> > > If it's platform-specific then wouldn't the pipeline handler have to\n> > > report it based on what platform it's running on? So we can't really\n> > >hardcode a constant. Maybe putting it in properties is indeed better.\n> >\n> > As far as I get it, this property applies to pipelines with an ISP (no\n> > uvc, no simple) and those pipelines by definition runs on a single\n> > platform, don't they ?\n>\n> Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n> but out-of-tree kernel patches)\n>\n> >\n> > >\n> > > >\n> > > > >\n> > > > > Need some thinking ...\n> > > > >\n> > > > > But none the less, this is a something that can be done on top\n> with wider\n> > > > > discussion / context on PipelineDepth. It shouldn't necessarily\n> block the\n> > > > > series.\n> > > >\n> > > > My whole point is that I think we should find a better mechanism for\n> > > > registering pipeline depth compared to having pipeline handlers call\n> > > > setMaxQueueRequests() as it requires an explicit action that can\n> > > > easily be overlooked, while we want the queue overflow protection to\n> > >\n> > > I mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\n> > > and then force *all* pipeline handlers to explictly set it, even if to\n> > > zero. Then it becomes really easy to catch.\n> >\n> > The property doesn't make much sense for UVC, right ?\n> >\n> > Not all pipelines will use libipa and have an FCQ (RPi in example at\n> > the moment). I don't think forcing all of them is actually necessary.\n>\n> Hm, that's true. No initial invalid value then.\n>\n> >\n> > But the pipelines that do need to throttle the number of requests in\n> > flight with the help of the PipelineHandler base class should ideally\n> > get it for free by simply registering the property ?\n>\n> Yeah that does sound more nice.\n>\n> >\n> > >\n> > > Or note it down as a compliancy thing and have lc-compliance test for\n> > > it. I presume this will become a required property to be exposed\n> anyway.\n> > >\n> > > In any case, the goal is to report to the user the \"maximum number of\n> > > requests that are allowed to be queued in the camera\", right? So it'll\n> > > have to go through properties anyway. Perhaps then it's best to expose\n> > > it via properties (as it's already the interface we have for exposing\n> > > properties to the user), and then the base PipelineHandler can cache it\n> > > for the queuing guard. That would get rid of the special explicit call,\n> > > and would still achieve the goal.\n> >\n> > That would be my preference. There might be a discussions if\n> > properties::PipelineDepth is the right property to report it or it\n> > will conflict with Android's pipeline depth, which is different.\n>\n> +1\n>\n> Although to me this sounds separate from PipelineDepth, because my\n> understanding is that that is the maximum amount of Requests that can be\n> in-flight, while what we want here is that plus some extra that the\n> application is allowed to \"overqueue\" that the pipeline handler will\n> automagically handle internally.\n>\n\nMy understanding of PipelineDepth is in-line with the above - it gives a\nminimum\nnumber of in-flight requests needed for the pipeline handler. This new\nproperty must give an upper bound on the number of in-flight requests the\npipeline\nhandler can handle. For the RPi case, the min is 1 and max is unbounded.\n\nI can see a possible bug(?) introduced in this series however. Say a\npipeline\nhandler sets maxQueueRequest to 16. An application comes along and queues\n20\nrequests and then just waits for completion of those 20 requests. The\nPipelineHandler base class calls queueRequestDevice for the first 16\nrequests,\nbut nothing that I can see will trigger queuing of the last 4 requests once\nthe\npipeline handler has completed the first N requests. Unless I've missed\nsomething?\n\nRegards,\nNaush\n\n\n> Am I misunderstanding something?\n>\n>\n> Paul\n>\n> >\n> > >\n> > > I suppose then we'd actually need a compliance test for the property,\n> > > but I guess we have a few more of those anyway.\n> > >\n> > > My two cents.\n> >\n> > Thanks ;)\n> > j\n> >\n> > >\n> > >\n> > > Paul\n> > >\n> > > > happen transparently if a pipeline handler register\n> > > > properties::PipelineDepth ? Am I overthinking this ?\n> > > >\n> > > >\n> > > > > >\n> > > > > > Sorry for not having been clear\n> > > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > > Also, can this feature be tested in test/ ? Probably not, as\n> it\n> > > > > > > > requires a compliant pipeline handler ? Maybe vimc can be\n> instrumented\n> > > > > > > I have been thinking about that as well. It's shouldn't be too\n> difficult.\n> > > > > > > > ?\n> > > > > > > >\n> > > > > > > > > The pipeline handler is now equipped with the\n> responsibility of not to\n> > > > > > > > > over queue the requests to the underlying layers. The\n> derived\n> > > > > > > > > pipeline handler (or even IPA) can also have various kind\n> of requests\n> > > > > > > > > queue(arrays, queues or ring-buffer) hence, it might be an\n> issue where\n> > > > > > > > > the application queues the requests at a rate where these\n> kind of\n> > > > > > > > > queues can over-flow. The patch ensures that the base\n> PipelineHandler\n> > > > > > > > > will never let the requests overflow to underlying layers,\n> once the\n> > > > > > > > > derived pipeline handler sets its maximum capacity of\n> handling\n> > > > > > > > > in-flight requests using\n> PipelineHandler::setMaxQueueRequests().\n> > > > > > > > >\n> > > > > > > > > The queue request management introduced in the pipeline\n> handler base\n> > > > > > > > > class is now used by the IPU3 and RkISP1 pipeline\n> handlers. This will\n> > > > > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in\n> these two\n> > > > > > > > > pipeline handlers.\n> > > > > > > > >\n> > > > > > > > > Signed-off-by: Umang Jain \n> > > > > > > > > ---\n> > > > > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > > > src/libcamera/pipeline_handler.cpp | 51\n> ++++++++++++++++++-\n> > > > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > > > >\n> > > > > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h\n> b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > > > protected:\n> > > > > > > > > void registerCamera(std::shared_ptr\n> camera);\n> > > > > > > > > void hotplugMediaDevice(MediaDevice *media);\n> > > > > > > > > + void setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > > > >\n> > > > > > > > > virtual int queueRequestDevice(Camera *camera,\n> Request *request) = 0;\n> > > > > > > > > virtual void stopDevice(Camera *camera) = 0;\n> > > > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > > > Mutex lock_;\n> > > > > > > > > unsigned int useCount_\n> LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > > > >\n> > > > > > > > > + uint32_t maxQueueRequests_;\n> > > > > > > > > + uint32_t requestsQueueCounter_;\n> > > > > > > > > +\n> > > > > > > > > friend class PipelineHandlerFactoryBase;\n> > > > > > > > > };\n> > > > > > > > >\n> > > > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status\n> IPU3CameraConfiguration::validate()\n> > > > > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager\n> *manager)\n> > > > > > > > > : PipelineHandler(manager),\n> cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > > > {\n> > > > > > > > > + setMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > std::unique_ptr\n> > > > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status\n> RkISP1CameraConfiguration::validate()\n> > > > > > > > >\n> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > > > : PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > > > {\n> > > > > > > > > +\n> setMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > std::unique_ptr\n> > > > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp\n> b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > index cfade490..103f9db0 100644\n> > > > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > > > * through the PipelineHandlerFactoryBase::create()\n> function.\n> > > > > > > > > */\n> > > > > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > > > > - : manager_(manager), useCount_(0)\n> > > > > > > > > + : manager_(manager), useCount_(0),\n> maxQueueRequests_(0),\n> > > > > > > > > + requestsQueueCounter_(0)\n> > > > > > > > > {\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > @@ -428,6 +429,9 @@ void\n> PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > > > Camera::Private *data = camera->_d();\n> > > > > > > > > data->queuedRequests_.push_back(request);\n> > > > > > > > >\n> > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > + requestsQueueCounter_++;\n> > > > > > > > > +\n> > > > > > > > > request->_d()->sequence_ =\n> data->requestSequence_++;\n> > > > > > > > >\n> > > > > > > > > if (request->_d()->cancelled_) {\n> > > > > > > > > @@ -455,6 +459,10 @@ void\n> PipelineHandler::doQueueRequests()\n> > > > > > > > > if (!request->_d()->prepared_)\n> > > > > > > > > break;\n> > > > > > > > >\n> > > > > > > > > + if (maxQueueRequests_ &&\n> > > > > > > > > + requestsQueueCounter_ >=\n> maxQueueRequests_)\n> > > > > > > > > + break;\n> > > > > > > > > +\n> > > > > > > > > doQueueRequest(request);\n> > > > > > > > > waitingRequests_.pop();\n> > > > > > > > > }\n> > > > > > > > > @@ -531,6 +539,9 @@ void\n> PipelineHandler::completeRequest(Request *request)\n> > > > > > > > > ASSERT(!req->hasPendingBuffers());\n> > > > > > > > > data->queuedRequests_.pop_front();\n> > > > > > > > > camera->requestComplete(req);\n> > > > > > > > > +\n> > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > + requestsQueueCounter_--;\n> > > > > > > > > }\n> > > > > > > > > }\n> > > > > > > > >\n> > > > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > > > * constant for the whole lifetime of the pipeline\n> handler.\n> > > > > > > > > */\n> > > > > > > > >\n> > > > > > > > > +/**\n> > > > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > > > + * \\brief Maximum number of in-flight requests that can\n> be queued\n> > > > > > > > > + *\n> > > > > > > > > + * A hardware can handle a certain number of maximum\n> requests at a given\n> > > > > > > > > + * point. If such a constraint exists, set\n> maxQueueRequests_ via\n> > > > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline\n> handler.\n> > > > > > > > > + *\n> > > > > > > > > + * The derived pipeline handler can choose not to define\n> such constraint as\n> > > > > > > > > + * well. In that case, the derived pipeline handler can\n> avoid setting\n> > > > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_\n> and\n> > > > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > > > + */\n> > > > > > > > > +\n> > > > > > > > > +/**\n> > > > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > > > + * \\brief Number of requests queued to the underlying\n> hardware\n> > > > > > > > > + *\n> > > > > > > > > + * If \\a setMaxQueueRequests() is set by the derived\n> pipeline handler,\n> > > > > > > > > + * requestsQueueCounter_ reflects the number of requests\n> queued\n> > > > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > > > + */\n> > > > > > > > > +\n> > > > > > > > > +/**\n> > > > > > > > > + * \\brief Sets the maximum number of requests that can be\n> queued\n> > > > > > > > > + * \\param[in] maxRequests Maximum number of in-flight\n> requests\n> > > > > > > > > + *\n> > > > > > > > > + * A hardware can handle a certain number of requests at\n> a given point.\n> > > > > > > > > + * This function sets the maximum number of in-flight\n> requests that can\n> > > > > > > > > + * be queued to the hardware by the pipeline handler.\n> Each derived pipeline\n> > > > > > > > > + * handler should set the maximum number of in-flight\n> requests it can handle\n> > > > > > > > > + * at a given point using this function, if at all such a\n> constraint exists.\n> > > > > > > > > + */\n> > > > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t\n> maxRequests)\n> > > > > > > > > +{\n> > > > > > > > > + maxQueueRequests_ = maxRequests;\n> > > > > > > > > +}\n> > > > > > > > > +\n> > > > > > > > > /**\n> > > > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > > > --\n> > > > > > > > > 2.38.1\n> > > > > > > > >\n> > > > >\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A87D2C3200\n\tfor ;\n\tThu, 15 Dec 2022 09:44:14 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 07EC46339D;\n\tThu, 15 Dec 2022 10:44:14 +0100 (CET)","from mail-vk1-xa33.google.com (mail-vk1-xa33.google.com\n\t[IPv6:2607:f8b0:4864:20::a33])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4873760410\n\tfor ;\n\tThu, 15 Dec 2022 10:44:12 +0100 (CET)","by mail-vk1-xa33.google.com with SMTP id q7so4437894vka.7\n\tfor ;\n\tThu, 15 Dec 2022 01:44:12 -0800 (PST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671097454;\n\tbh=phI9ynGTVj8traOJ6VIe9nEOX1I+UqA6xV+1CnzctS8=;\n\th=References:In-Reply-To:Date:To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=O8zZaWGfjpOVo/YjOoafhHM1FUOvi+jl/ZKIzB8wL3iFUkf/xTTY0Nlhh6UhD4BGG\n\tlY0yHjjp4/DFIjuKEXO1LusX21Vpopqp/8jUtXlXbPxhMWMTm6pDajY9QTahWp6p2+\n\tqJY6u4EB+4NrtAVYqu57BoF3OhOxXLfk76Z4mw+TGgWswquj/12rJwT5dbYeXI+2Ga\n\tUlaIyhx32PxzZx/wlIS81xcrLpdBBhyYTLWVoH9O0a9WZvD43Jnpc6zEteQXmFdyyW\n\t7ObZOL8gwoyJ1Mg8R8TqI5iX/TxjZ6+1MSpvxei4qsFuxL4bIgW7Lv3W3UeRCB3Urv\n\tm8TFadmp4SeDQ==","v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=qqSLIPzYnFLA/Xmf9HAzH/4LDNG78HKqmBloUEO2qoQ=;\n\tb=GvodLry5O6BXKfez0fe1AecP0xgOX3xjEvDBL/aUFmrTyze5O8yqA7IVoc5pZBJPvz\n\tUodD3/CA5IN62etOl+jRLw4QxtMnrYOBrM0SASrsZMXLVr1xO4/diQyedABu+5oRHLCo\n\txUzkbi+cQBFYDPF8Qe/ucUs6ms+O1pPemXV2UP0HsLRDyzYWHjgOcnWtXikXooT2BR0V\n\tjcsHLOxFV3zc/LJf2wY7rtgjEHaczZsMbEkPY2D5oWuBics1I0ydh8vNtYkSrXVe0ydb\n\tJZI8tsTQ0S090yaOWyI2LeLPY7DoT2NuZYT86hIVHbf+U1Qkglzx2t3kk4ba8tMED31R\n\tXxVw=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=raspberrypi.com\n\theader.i=@raspberrypi.com\n\theader.b=\"GvodLry5\"; dkim-atps=neutral","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=qqSLIPzYnFLA/Xmf9HAzH/4LDNG78HKqmBloUEO2qoQ=;\n\tb=RIxcQyX7rtbGZipUWhbW2XRc/pOdwhwB3YsiEh/6WnD9/BUc2Aqs0P2K082OzSYmwl\n\twEu0zEUvcbSVe8xjTmXVjEue/Wv53zWPTSKzrKqsVOwWIis/9XPowmFQulEv+QJEzgNN\n\to8MngprK1Ni6xL1GuzcpFifz9gXrdeu6WPcLwn5zVQDb1pFb8eb9PRdNyAm79v1Cx8t/\n\tKkSBDf79SyJMdGVjrD1AP/uthV8hK+zjWqp43D+VkzQGG6+kkRafUDfigpjATjABCUb0\n\tpDcneIpa+kzP2gKz5j2d2E39EPM+MjFDlgUxu3m6+G6JOWWkZA3IUIm9KEJYORxQx1az\n\trCmw==","X-Gm-Message-State":"ANoB5pnUiP2A3oq35/q1osoKpY9auu1bwYJ89z8wIy1Q7FFK4rr7zUfu\n\tSfsjkPQUTVlnra1GfDO0mMSM5D3Tcpi1Taq0D1CvOeESEoJSCW8U","X-Google-Smtp-Source":"AA0mqf45aBeUjtj4Kgpq3Az3lKKD1n2pe8kaXrEuG9OCxpBIkex4jbMfBvcfRp3bXC343XZSLKe2zB5vJD5/ZIM0Jmc=","X-Received":"by 2002:a05:6122:d04:b0:3bd:6db7:262b with SMTP id\n\taz4-20020a0561220d0400b003bd6db7262bmr15761987vkb.27.1671097450904;\n\tThu, 15 Dec 2022 01:44:10 -0800 (PST)","MIME-Version":"1.0","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221212150256.69004-3-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t","In-Reply-To":"","Date":"Thu, 15 Dec 2022 09:43:55 +0000","Message-ID":"","To":"Paul Elder ","Content-Type":"multipart/alternative; boundary=\"000000000000f2524005efdab1d4\"","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Naushir Patuck via libcamera-devel\n\t","Reply-To":"Naushir Patuck ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26085,"web_url":"https://patchwork.libcamera.org/comment/26085/","msgid":"<167109791908.9133.17756312340732843792@Monstersaurus>","date":"2022-12-15T09:51:59","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Naushir Patuck via libcamera-devel (2022-12-15 09:43:55)\n> Hi all,\n> \n> On Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\n> libcamera-devel@lists.libcamera.org> wrote:\n> \n> > Hi Jacopo,\n> >\n> > On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> > > Hi Paul\n> > >\n> > > On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > > > Hi,\n> > > >\n> > > > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n> > libcamera-devel wrote:\n> > > > > Hi Umang\n> > > > >\n> > > > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > > > Hi Jacopo\n> > > > > >\n> > > > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > > > Hi Umang\n> > > > > > >\n> > > > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > > > Hi Jacopo,\n> > > > > > > >\n> > > > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > > > Hi Umang,\n> > > > > > > > >\n> > > > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n> > libcamera-devel wrote:\n> > > > > > > > > > This patch introduces a new constraint in the pipeline\n> > handler base\n> > > > > > > > > > class which allows various derived pipeline handlers to\n> > set and\n> > > > > > > > > > restrict the number of maximum in-flight requests that can\n> > be queued to\n> > > > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > > > It's my understanding that the max number of in-flight\n> > requests is\n> > > > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > > > Indeed, that's why understanding as well. The\n> > properties::PipelineDepth\n> > > > > > > > should be set by the pipeline-handlers.\n> > > >\n> > > > (Besides the fact that I can't find properties::PipelineDepth)\n> > > > controls::PipelineDepth says that it \"Specifies the number of pipeline\n> > > > stages the frame went through from when it was exposed to when the\n> > final\n> > > > completed result was available to the framework.\", which to me sounds\n> > > > like the *minimum* amount of in-flight requests.\n> > >\n> > > It's a draft control copied in to please Android.\n> > > My understanding was that it had to be made a non-mutable property\n> >\n> > Ah, indeed it says that :)\n> >\n> > >\n> > > >\n> > > > I thought that here we're defining the *maximum* allowed in-flight\n> > > > requests. Not that all of the requests have to actually be doing\n> > > > something in hardware, but that it's the amount of requests the the\n> > > > pipeline handler is capable of handling internally without overflowing.\n> > > >\n> > > > > > > >\n> > > > > > > > We need to identify and atleast agree upon the correct\n> > PipelineDepth for the\n> > > > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as well.\n> > Some grey\n> > > > > > > > areas there as well\n> > > > > > > I think 16 is fine for now. At least doesn't change what we\n> > already\n> > > > > > > have today\n> > > > > > >\n> > > > > > > > > Can't we access that property at queue request time, or\n> > maybe better we\n> > > > > > > > > could cache it at camera registration time, since it's\n> > constant ?\n> > > > > > > > If we intent to use the properties::PipelineDepth, as the size\n> > of the\n> > > > > > > > FCQueue - it needs to exist / set before IPA is constructed.\n> > > > > > > >\n> > > > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > > > >\n> > > > > > > The property can still be defined in the IPA header as IPA can\n> > access\n> > > > > > > it safely.\n> > > > > >\n> > > > > > You mean via a global const variable like it's done in Patch 1/2 ?\n> > > > >\n> > > > > Yes, like in patch 1/2\n> > > > >\n> > > > >\n> > > > > > >\n> > > > > > > Pipeline Handlers will have to register the property -before- the\n> > > > > > > camera is registered, but looking at IPU3 that's already the\n> > case now\n> > > > > > > (data->properties_ gets populated before calling\n> > registerCamera())\n> > > > > > >\n> > > > > > >\n> > > > > > > > So I think queue request time is quite late there. Camera\n> > registeration time\n> > > > > > > > might be possible with some changes (looking at\n> > > > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > > > What I meant is either\n> > > > > > >\n> > > > > > > 1) At PipelineHandler::queueRequest() time, the PipelineHandler\n> > base\n> > > > > > > class accesses Camera::properties() and fetches\n> > > > > > > properties::PipelineDepth\n> > > > > >\n> > > > > > I think it's a over-kill here. queueRequest() is a recurring call\n> > and I\n> > > > > > don't want fetching of PipelineDepth here.\n> > > >\n> > > > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > > > correct, but on the other hand it does indeed seem overkill.\n> > > >\n> > > > >\n> > > > > While I consider a lookup on an unordered_map<> of a few items not\n> > > > > that expensive, I agree it's in an hot path, and it would be better\n> > to\n> > > > > cache the value somewhere. Caching it inside Camera or\n> > Camera::Private\n> > > > > is a duplication, as it's already part of\n> > > > > Camera::Private::properties_, but I would be fine with the\n> > > > > PipelineHandler base class doing something like this (as proposed in\n> > > > > my previous 2) point below)\n> > > > >\n> > > > > void PipelineHandler::registerCamera(std::shared_ptr camera)\n> > > > > {\n> > > > >\n> > > > > /* Cache the pipeline depth in the Camera private data. */\n> > > > > const auto &pipelineDepth =\n> > camera->properties().get(properties::PipelineDepth);\n> > > > >\n> > > > > Camera::Private *data = camera->_d();\n> > > > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > > > >\n> > > > > ...\n> > > > > }\n> > > > >\n> > > > > Otherwise the PipelineHandler base class can be added with a class\n> > > > > member, and each registred camera will overwrite that value, it's\n> > less\n> > > > > nice, but all cameras from the same PH instance will have the same\n> > > > > pipeline depth, so...\n> > > >\n> > > > Perhaps caching would be a good middle ground.\n> > > >\n> > > > >\n> > > > >\n> > > > > > > 2) At PipelineHandler::registerCamera() time the property value\n> > is\n> > > > > > > cached, but this would require to cache per-camera data in\n> > > > > > > PipelineHandler, something we don't have, or to populate a class\n> > > > > > > member of Camera::Private for that purpose..\n> > > > > >\n> > > > > > That's interesting because I initially didn't think of multiple\n> > cameras\n> > > > > > registered with one instance of pipeline-handler.\n> > > > > > It also makes me wonder -\n> > > > > >\n> > > > > > - Is properties::PipelineDepth Camera specific or\n> > platform-specific ? Seems\n> > > > > > now it's Camera specific\n> > > > > > - Then defining it IPA headers won't be a good option, in cases\n> > where\n> > > > > > PipelineDepth for different cameras differ.\n> > > > >\n> > > > > Why camera specific ? Each camera will indeed register it as a\n> > > > > property for applications to retrieve it, but the\n> > > > > value depends on the platform afaict and it can be a compile-time\n> > constant\n> > > > > defined by the IPA headers. Even better, the same compile-time\n> > > > > constant will be used to size the FCQ.\n> > > >\n> > > > If it's platform-specific then wouldn't the pipeline handler have to\n> > > > report it based on what platform it's running on? So we can't really\n> > > >hardcode a constant. Maybe putting it in properties is indeed better.\n> > >\n> > > As far as I get it, this property applies to pipelines with an ISP (no\n> > > uvc, no simple) and those pipelines by definition runs on a single\n> > > platform, don't they ?\n> >\n> > Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n> > but out-of-tree kernel patches)\n> >\n> > >\n> > > >\n> > > > >\n> > > > > >\n> > > > > > Need some thinking ...\n> > > > > >\n> > > > > > But none the less, this is a something that can be done on top\n> > with wider\n> > > > > > discussion / context on PipelineDepth. It shouldn't necessarily\n> > block the\n> > > > > > series.\n> > > > >\n> > > > > My whole point is that I think we should find a better mechanism for\n> > > > > registering pipeline depth compared to having pipeline handlers call\n> > > > > setMaxQueueRequests() as it requires an explicit action that can\n> > > > > easily be overlooked, while we want the queue overflow protection to\n> > > >\n> > > > I mean, we could initialize kMaxFrameContexts to an invalid value (-1?)\n> > > > and then force *all* pipeline handlers to explictly set it, even if to\n> > > > zero. Then it becomes really easy to catch.\n> > >\n> > > The property doesn't make much sense for UVC, right ?\n> > >\n> > > Not all pipelines will use libipa and have an FCQ (RPi in example at\n> > > the moment). I don't think forcing all of them is actually necessary.\n> >\n> > Hm, that's true. No initial invalid value then.\n> >\n> > >\n> > > But the pipelines that do need to throttle the number of requests in\n> > > flight with the help of the PipelineHandler base class should ideally\n> > > get it for free by simply registering the property ?\n> >\n> > Yeah that does sound more nice.\n> >\n> > >\n> > > >\n> > > > Or note it down as a compliancy thing and have lc-compliance test for\n> > > > it. I presume this will become a required property to be exposed\n> > anyway.\n> > > >\n> > > > In any case, the goal is to report to the user the \"maximum number of\n> > > > requests that are allowed to be queued in the camera\", right? So it'll\n> > > > have to go through properties anyway. Perhaps then it's best to expose\n> > > > it via properties (as it's already the interface we have for exposing\n> > > > properties to the user), and then the base PipelineHandler can cache it\n> > > > for the queuing guard. That would get rid of the special explicit call,\n> > > > and would still achieve the goal.\n> > >\n> > > That would be my preference. There might be a discussions if\n> > > properties::PipelineDepth is the right property to report it or it\n> > > will conflict with Android's pipeline depth, which is different.\n> >\n> > +1\n> >\n> > Although to me this sounds separate from PipelineDepth, because my\n> > understanding is that that is the maximum amount of Requests that can be\n> > in-flight, while what we want here is that plus some extra that the\n> > application is allowed to \"overqueue\" that the pipeline handler will\n> > automagically handle internally.\n> >\n> \n> My understanding of PipelineDepth is in-line with the above - it gives a\n> minimum\n> number of in-flight requests needed for the pipeline handler. This new\n> property must give an upper bound on the number of in-flight requests the\n> pipeline\n> handler can handle. For the RPi case, the min is 1 and max is unbounded.\n> \n> I can see a possible bug(?) introduced in this series however. Say a\n> pipeline\n> handler sets maxQueueRequest to 16. An application comes along and queues\n> 20\n> requests and then just waits for completion of those 20 requests. The\n> PipelineHandler base class calls queueRequestDevice for the first 16\n> requests,\n> but nothing that I can see will trigger queuing of the last 4 requests once\n> the\n> pipeline handler has completed the first N requests. Unless I've missed\n> something?\n\nI think I saw Jacopo confirm the same in a parallel reply here, but I\nunderstood this to be something we shouldn't expose to applications.\nApplications can always choose to queue an unlimited number of requests\nto a Camera (within the limits of the universe) ... but it's the\npipeline handlers responsibility to ensure that requests are not queued\nbeyond the capabilities of the IPA. As such - requests may stay 'queued'\nin the pipeline handler or CameraData object until the IPA is ready to\nprocess the next one.\n\n--\nKieran\n\n\n\n> \n> Regards,\n> Naush\n> \n> \n> > Am I misunderstanding something?\n> >\n> >\n> > Paul\n> >\n> > >\n> > > >\n> > > > I suppose then we'd actually need a compliance test for the property,\n> > > > but I guess we have a few more of those anyway.\n> > > >\n> > > > My two cents.\n> > >\n> > > Thanks ;)\n> > > j\n> > >\n> > > >\n> > > >\n> > > > Paul\n> > > >\n> > > > > happen transparently if a pipeline handler register\n> > > > > properties::PipelineDepth ? Am I overthinking this ?\n> > > > >\n> > > > >\n> > > > > > >\n> > > > > > > Sorry for not having been clear\n> > > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > > Also, can this feature be tested in test/ ? Probably not, as\n> > it\n> > > > > > > > > requires a compliant pipeline handler ? Maybe vimc can be\n> > instrumented\n> > > > > > > > I have been thinking about that as well. It's shouldn't be too\n> > difficult.\n> > > > > > > > > ?\n> > > > > > > > >\n> > > > > > > > > > The pipeline handler is now equipped with the\n> > responsibility of not to\n> > > > > > > > > > over queue the requests to the underlying layers. The\n> > derived\n> > > > > > > > > > pipeline handler (or even IPA) can also have various kind\n> > of requests\n> > > > > > > > > > queue(arrays, queues or ring-buffer) hence, it might be an\n> > issue where\n> > > > > > > > > > the application queues the requests at a rate where these\n> > kind of\n> > > > > > > > > > queues can over-flow. The patch ensures that the base\n> > PipelineHandler\n> > > > > > > > > > will never let the requests overflow to underlying layers,\n> > once the\n> > > > > > > > > > derived pipeline handler sets its maximum capacity of\n> > handling\n> > > > > > > > > > in-flight requests using\n> > PipelineHandler::setMaxQueueRequests().\n> > > > > > > > > >\n> > > > > > > > > > The queue request management introduced in the pipeline\n> > handler base\n> > > > > > > > > > class is now used by the IPU3 and RkISP1 pipeline\n> > handlers. This will\n> > > > > > > > > > prevent over-writing of frame contexts (i.e. FCQueue) in\n> > these two\n> > > > > > > > > > pipeline handlers.\n> > > > > > > > > >\n> > > > > > > > > > Signed-off-by: Umang Jain \n> > > > > > > > > > ---\n> > > > > > > > > > include/libcamera/internal/pipeline_handler.h | 4 ++\n> > > > > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > > > > src/libcamera/pipeline_handler.cpp | 51\n> > ++++++++++++++++++-\n> > > > > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > > > > >\n> > > > > > > > > > diff --git a/include/libcamera/internal/pipeline_handler.h\n> > b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > > > > protected:\n> > > > > > > > > > void registerCamera(std::shared_ptr\n> > camera);\n> > > > > > > > > > void hotplugMediaDevice(MediaDevice *media);\n> > > > > > > > > > + void setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > > > > >\n> > > > > > > > > > virtual int queueRequestDevice(Camera *camera,\n> > Request *request) = 0;\n> > > > > > > > > > virtual void stopDevice(Camera *camera) = 0;\n> > > > > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > > > > Mutex lock_;\n> > > > > > > > > > unsigned int useCount_\n> > LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > > > > >\n> > > > > > > > > > + uint32_t maxQueueRequests_;\n> > > > > > > > > > + uint32_t requestsQueueCounter_;\n> > > > > > > > > > +\n> > > > > > > > > > friend class PipelineHandlerFactoryBase;\n> > > > > > > > > > };\n> > > > > > > > > >\n> > > > > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status\n> > IPU3CameraConfiguration::validate()\n> > > > > > > > > > PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager\n> > *manager)\n> > > > > > > > > > : PipelineHandler(manager),\n> > cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > > > > {\n> > > > > > > > > > + setMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > > > > }\n> > > > > > > > > >\n> > > > > > > > > > std::unique_ptr\n> > > > > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status\n> > RkISP1CameraConfiguration::validate()\n> > > > > > > > > >\n> > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > > > > : PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > > > > {\n> > > > > > > > > > +\n> > setMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > > > > }\n> > > > > > > > > >\n> > > > > > > > > > std::unique_ptr\n> > > > > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp\n> > b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > index cfade490..103f9db0 100644\n> > > > > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > > > > * through the PipelineHandlerFactoryBase::create()\n> > function.\n> > > > > > > > > > */\n> > > > > > > > > > PipelineHandler::PipelineHandler(CameraManager *manager)\n> > > > > > > > > > - : manager_(manager), useCount_(0)\n> > > > > > > > > > + : manager_(manager), useCount_(0),\n> > maxQueueRequests_(0),\n> > > > > > > > > > + requestsQueueCounter_(0)\n> > > > > > > > > > {\n> > > > > > > > > > }\n> > > > > > > > > >\n> > > > > > > > > > @@ -428,6 +429,9 @@ void\n> > PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > > > > Camera::Private *data = camera->_d();\n> > > > > > > > > > data->queuedRequests_.push_back(request);\n> > > > > > > > > >\n> > > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > > + requestsQueueCounter_++;\n> > > > > > > > > > +\n> > > > > > > > > > request->_d()->sequence_ =\n> > data->requestSequence_++;\n> > > > > > > > > >\n> > > > > > > > > > if (request->_d()->cancelled_) {\n> > > > > > > > > > @@ -455,6 +459,10 @@ void\n> > PipelineHandler::doQueueRequests()\n> > > > > > > > > > if (!request->_d()->prepared_)\n> > > > > > > > > > break;\n> > > > > > > > > >\n> > > > > > > > > > + if (maxQueueRequests_ &&\n> > > > > > > > > > + requestsQueueCounter_ >=\n> > maxQueueRequests_)\n> > > > > > > > > > + break;\n> > > > > > > > > > +\n> > > > > > > > > > doQueueRequest(request);\n> > > > > > > > > > waitingRequests_.pop();\n> > > > > > > > > > }\n> > > > > > > > > > @@ -531,6 +539,9 @@ void\n> > PipelineHandler::completeRequest(Request *request)\n> > > > > > > > > > ASSERT(!req->hasPendingBuffers());\n> > > > > > > > > > data->queuedRequests_.pop_front();\n> > > > > > > > > > camera->requestComplete(req);\n> > > > > > > > > > +\n> > > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > > + requestsQueueCounter_--;\n> > > > > > > > > > }\n> > > > > > > > > > }\n> > > > > > > > > >\n> > > > > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > > > > * constant for the whole lifetime of the pipeline\n> > handler.\n> > > > > > > > > > */\n> > > > > > > > > >\n> > > > > > > > > > +/**\n> > > > > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > > > > + * \\brief Maximum number of in-flight requests that can\n> > be queued\n> > > > > > > > > > + *\n> > > > > > > > > > + * A hardware can handle a certain number of maximum\n> > requests at a given\n> > > > > > > > > > + * point. If such a constraint exists, set\n> > maxQueueRequests_ via\n> > > > > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline\n> > handler.\n> > > > > > > > > > + *\n> > > > > > > > > > + * The derived pipeline handler can choose not to define\n> > such constraint as\n> > > > > > > > > > + * well. In that case, the derived pipeline handler can\n> > avoid setting\n> > > > > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a maxQueueRequests_\n> > and\n> > > > > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > > > > + */\n> > > > > > > > > > +\n> > > > > > > > > > +/**\n> > > > > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > > > > + * \\brief Number of requests queued to the underlying\n> > hardware\n> > > > > > > > > > + *\n> > > > > > > > > > + * If \\a setMaxQueueRequests() is set by the derived\n> > pipeline handler,\n> > > > > > > > > > + * requestsQueueCounter_ reflects the number of requests\n> > queued\n> > > > > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > > > > + */\n> > > > > > > > > > +\n> > > > > > > > > > +/**\n> > > > > > > > > > + * \\brief Sets the maximum number of requests that can be\n> > queued\n> > > > > > > > > > + * \\param[in] maxRequests Maximum number of in-flight\n> > requests\n> > > > > > > > > > + *\n> > > > > > > > > > + * A hardware can handle a certain number of requests at\n> > a given point.\n> > > > > > > > > > + * This function sets the maximum number of in-flight\n> > requests that can\n> > > > > > > > > > + * be queued to the hardware by the pipeline handler.\n> > Each derived pipeline\n> > > > > > > > > > + * handler should set the maximum number of in-flight\n> > requests it can handle\n> > > > > > > > > > + * at a given point using this function, if at all such a\n> > constraint exists.\n> > > > > > > > > > + */\n> > > > > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t\n> > maxRequests)\n> > > > > > > > > > +{\n> > > > > > > > > > + maxQueueRequests_ = maxRequests;\n> > > > > > > > > > +}\n> > > > > > > > > > +\n> > > > > > > > > > /**\n> > > > > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > > > > --\n> > > > > > > > > > 2.38.1\n> > > > > > > > > >\n> > > > > >\n> >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 288EEC31E9\n\tfor ;\n\tThu, 15 Dec 2022 09:52:04 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 8365C6339D;\n\tThu, 15 Dec 2022 10:52:03 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 7EC4860410\n\tfor ;\n\tThu, 15 Dec 2022 10:52:02 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id EA2FD327;\n\tThu, 15 Dec 2022 10:52:01 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671097923;\n\tbh=K5NeCNAxu1DqskZ4Jlb85N+c9KxlKecJHYuGsCatC50=;\n\th=In-Reply-To:References:To:Date:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=y/4WoqpDjofv/SHmPzCP0yJTccsBE/z/fuKbdtWZbyDYnHVn16RXQxybwrzJ1nygE\n\tRjZ31tHOE9Tcp0j/8g0MEoadrexntQXDN5UjePoJ0r+WKtUG//f6f+FrdARVgHRFVU\n\tScAb9mOM1PTCwx4w6uWHulX18d6VcDwiZyIDKA9jGEYc/5MD1tQLlCw5LKcqD2qXNn\n\tk2ANwiB/jkeQBWtCZAOumz82XFSJ2WTqLluBFGqb802AvAgXEHRW/vEr4E+cathNiW\n\tJG9SeiLbc3iFYF7UiM5Vg1SqZodidtYNvoI5q4ql9nLNj12+zqORWVI1EqBJ+Xa715\n\tpDDXXlGqxeNhQ==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671097922;\n\tbh=K5NeCNAxu1DqskZ4Jlb85N+c9KxlKecJHYuGsCatC50=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=LLdHGTKtTeykHnIb5ZIh8HLv86DinYhsoFnhm3S9vikIAuuOtZsmbM33fsWG2Ig61\n\tqs4pkvpP3fP8FB5/GfYtrx2MnowBRUR2KSzln7NJ+WWGTmWhE0ihIC+3JS01A29G39\n\toLhboz2LpsJplgaPmj8OTcQbtITlsfvkTO+ngh3E="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"LLdHGTKt\"; dkim-atps=neutral","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t\n\t","To":"Naushir Patuck ,\n\tNaushir Patuck via libcamera-devel\n\t, \n\tPaul Elder ","Date":"Thu, 15 Dec 2022 09:51:59 +0000","Message-ID":"<167109791908.9133.17756312340732843792@Monstersaurus>","User-Agent":"alot/0.10","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Kieran Bingham via libcamera-devel\n\t","Reply-To":"Kieran Bingham ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26086,"web_url":"https://patchwork.libcamera.org/comment/26086/","msgid":"","date":"2022-12-15T09:59:51","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"On Thu, 15 Dec 2022 at 09:52, Kieran Bingham <\nkieran.bingham@ideasonboard.com> wrote:\n\n> Quoting Naushir Patuck via libcamera-devel (2022-12-15 09:43:55)\n> > Hi all,\n> >\n> > On Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\n> > libcamera-devel@lists.libcamera.org> wrote:\n> >\n> > > Hi Jacopo,\n> > >\n> > > On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> > > > Hi Paul\n> > > >\n> > > > On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > > > > Hi,\n> > > > >\n> > > > > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n> > > libcamera-devel wrote:\n> > > > > > Hi Umang\n> > > > > >\n> > > > > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > > > > Hi Jacopo\n> > > > > > >\n> > > > > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > > > > Hi Umang\n> > > > > > > >\n> > > > > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > > > > Hi Jacopo,\n> > > > > > > > >\n> > > > > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > > > > Hi Umang,\n> > > > > > > > > >\n> > > > > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n> > > libcamera-devel wrote:\n> > > > > > > > > > > This patch introduces a new constraint in the pipeline\n> > > handler base\n> > > > > > > > > > > class which allows various derived pipeline handlers to\n> > > set and\n> > > > > > > > > > > restrict the number of maximum in-flight requests that\n> can\n> > > be queued to\n> > > > > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > > > > It's my understanding that the max number of in-flight\n> > > requests is\n> > > > > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > > > > Indeed, that's why understanding as well. The\n> > > properties::PipelineDepth\n> > > > > > > > > should be set by the pipeline-handlers.\n> > > > >\n> > > > > (Besides the fact that I can't find properties::PipelineDepth)\n> > > > > controls::PipelineDepth says that it \"Specifies the number of\n> pipeline\n> > > > > stages the frame went through from when it was exposed to when the\n> > > final\n> > > > > completed result was available to the framework.\", which to me\n> sounds\n> > > > > like the *minimum* amount of in-flight requests.\n> > > >\n> > > > It's a draft control copied in to please Android.\n> > > > My understanding was that it had to be made a non-mutable property\n> > >\n> > > Ah, indeed it says that :)\n> > >\n> > > >\n> > > > >\n> > > > > I thought that here we're defining the *maximum* allowed in-flight\n> > > > > requests. Not that all of the requests have to actually be doing\n> > > > > something in hardware, but that it's the amount of requests the the\n> > > > > pipeline handler is capable of handling internally without\n> overflowing.\n> > > > >\n> > > > > > > > >\n> > > > > > > > > We need to identify and atleast agree upon the correct\n> > > PipelineDepth for the\n> > > > > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as\n> well.\n> > > Some grey\n> > > > > > > > > areas there as well\n> > > > > > > > I think 16 is fine for now. At least doesn't change what we\n> > > already\n> > > > > > > > have today\n> > > > > > > >\n> > > > > > > > > > Can't we access that property at queue request time, or\n> > > maybe better we\n> > > > > > > > > > could cache it at camera registration time, since it's\n> > > constant ?\n> > > > > > > > > If we intent to use the properties::PipelineDepth, as the\n> size\n> > > of the\n> > > > > > > > > FCQueue - it needs to exist / set before IPA is\n> constructed.\n> > > > > > > > >\n> > > > > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > > > > >\n> > > > > > > > The property can still be defined in the IPA header as IPA\n> can\n> > > access\n> > > > > > > > it safely.\n> > > > > > >\n> > > > > > > You mean via a global const variable like it's done in Patch\n> 1/2 ?\n> > > > > >\n> > > > > > Yes, like in patch 1/2\n> > > > > >\n> > > > > >\n> > > > > > > >\n> > > > > > > > Pipeline Handlers will have to register the property\n> -before- the\n> > > > > > > > camera is registered, but looking at IPU3 that's already the\n> > > case now\n> > > > > > > > (data->properties_ gets populated before calling\n> > > registerCamera())\n> > > > > > > >\n> > > > > > > >\n> > > > > > > > > So I think queue request time is quite late there. Camera\n> > > registeration time\n> > > > > > > > > might be possible with some changes (looking at\n> > > > > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > > > > What I meant is either\n> > > > > > > >\n> > > > > > > > 1) At PipelineHandler::queueRequest() time, the\n> PipelineHandler\n> > > base\n> > > > > > > > class accesses Camera::properties() and fetches\n> > > > > > > > properties::PipelineDepth\n> > > > > > >\n> > > > > > > I think it's a over-kill here. queueRequest() is a recurring\n> call\n> > > and I\n> > > > > > > don't want fetching of PipelineDepth here.\n> > > > >\n> > > > > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > > > > correct, but on the other hand it does indeed seem overkill.\n> > > > >\n> > > > > >\n> > > > > > While I consider a lookup on an unordered_map<> of a few items\n> not\n> > > > > > that expensive, I agree it's in an hot path, and it would be\n> better\n> > > to\n> > > > > > cache the value somewhere. Caching it inside Camera or\n> > > Camera::Private\n> > > > > > is a duplication, as it's already part of\n> > > > > > Camera::Private::properties_, but I would be fine with the\n> > > > > > PipelineHandler base class doing something like this (as\n> proposed in\n> > > > > > my previous 2) point below)\n> > > > > >\n> > > > > > void PipelineHandler::registerCamera(std::shared_ptr\n> camera)\n> > > > > > {\n> > > > > >\n> > > > > > /* Cache the pipeline depth in the Camera private data. */\n> > > > > > const auto &pipelineDepth =\n> > > camera->properties().get(properties::PipelineDepth);\n> > > > > >\n> > > > > > Camera::Private *data = camera->_d();\n> > > > > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > > > > >\n> > > > > > ...\n> > > > > > }\n> > > > > >\n> > > > > > Otherwise the PipelineHandler base class can be added with a\n> class\n> > > > > > member, and each registred camera will overwrite that value, it's\n> > > less\n> > > > > > nice, but all cameras from the same PH instance will have the\n> same\n> > > > > > pipeline depth, so...\n> > > > >\n> > > > > Perhaps caching would be a good middle ground.\n> > > > >\n> > > > > >\n> > > > > >\n> > > > > > > > 2) At PipelineHandler::registerCamera() time the property\n> value\n> > > is\n> > > > > > > > cached, but this would require to cache per-camera data in\n> > > > > > > > PipelineHandler, something we don't have, or to populate a\n> class\n> > > > > > > > member of Camera::Private for that purpose..\n> > > > > > >\n> > > > > > > That's interesting because I initially didn't think of multiple\n> > > cameras\n> > > > > > > registered with one instance of pipeline-handler.\n> > > > > > > It also makes me wonder -\n> > > > > > >\n> > > > > > > - Is properties::PipelineDepth Camera specific or\n> > > platform-specific ? Seems\n> > > > > > > now it's Camera specific\n> > > > > > > - Then defining it IPA headers won't be a good option, in cases\n> > > where\n> > > > > > > PipelineDepth for different cameras differ.\n> > > > > >\n> > > > > > Why camera specific ? Each camera will indeed register it as a\n> > > > > > property for applications to retrieve it, but the\n> > > > > > value depends on the platform afaict and it can be a compile-time\n> > > constant\n> > > > > > defined by the IPA headers. Even better, the same compile-time\n> > > > > > constant will be used to size the FCQ.\n> > > > >\n> > > > > If it's platform-specific then wouldn't the pipeline handler have\n> to\n> > > > > report it based on what platform it's running on? So we can't\n> really\n> > > > >hardcode a constant. Maybe putting it in properties is indeed\n> better.\n> > > >\n> > > > As far as I get it, this property applies to pipelines with an ISP\n> (no\n> > > > uvc, no simple) and those pipelines by definition runs on a single\n> > > > platform, don't they ?\n> > >\n> > > Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n> > > but out-of-tree kernel patches)\n> > >\n> > > >\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > Need some thinking ...\n> > > > > > >\n> > > > > > > But none the less, this is a something that can be done on top\n> > > with wider\n> > > > > > > discussion / context on PipelineDepth. It shouldn't necessarily\n> > > block the\n> > > > > > > series.\n> > > > > >\n> > > > > > My whole point is that I think we should find a better mechanism\n> for\n> > > > > > registering pipeline depth compared to having pipeline handlers\n> call\n> > > > > > setMaxQueueRequests() as it requires an explicit action that can\n> > > > > > easily be overlooked, while we want the queue overflow\n> protection to\n> > > > >\n> > > > > I mean, we could initialize kMaxFrameContexts to an invalid value\n> (-1?)\n> > > > > and then force *all* pipeline handlers to explictly set it, even\n> if to\n> > > > > zero. Then it becomes really easy to catch.\n> > > >\n> > > > The property doesn't make much sense for UVC, right ?\n> > > >\n> > > > Not all pipelines will use libipa and have an FCQ (RPi in example at\n> > > > the moment). I don't think forcing all of them is actually necessary.\n> > >\n> > > Hm, that's true. No initial invalid value then.\n> > >\n> > > >\n> > > > But the pipelines that do need to throttle the number of requests in\n> > > > flight with the help of the PipelineHandler base class should ideally\n> > > > get it for free by simply registering the property ?\n> > >\n> > > Yeah that does sound more nice.\n> > >\n> > > >\n> > > > >\n> > > > > Or note it down as a compliancy thing and have lc-compliance test\n> for\n> > > > > it. I presume this will become a required property to be exposed\n> > > anyway.\n> > > > >\n> > > > > In any case, the goal is to report to the user the \"maximum number\n> of\n> > > > > requests that are allowed to be queued in the camera\", right? So\n> it'll\n> > > > > have to go through properties anyway. Perhaps then it's best to\n> expose\n> > > > > it via properties (as it's already the interface we have for\n> exposing\n> > > > > properties to the user), and then the base PipelineHandler can\n> cache it\n> > > > > for the queuing guard. That would get rid of the special explicit\n> call,\n> > > > > and would still achieve the goal.\n> > > >\n> > > > That would be my preference. There might be a discussions if\n> > > > properties::PipelineDepth is the right property to report it or it\n> > > > will conflict with Android's pipeline depth, which is different.\n> > >\n> > > +1\n> > >\n> > > Although to me this sounds separate from PipelineDepth, because my\n> > > understanding is that that is the maximum amount of Requests that can\n> be\n> > > in-flight, while what we want here is that plus some extra that the\n> > > application is allowed to \"overqueue\" that the pipeline handler will\n> > > automagically handle internally.\n> > >\n> >\n> > My understanding of PipelineDepth is in-line with the above - it gives a\n> > minimum\n> > number of in-flight requests needed for the pipeline handler. This new\n> > property must give an upper bound on the number of in-flight requests the\n> > pipeline\n> > handler can handle. For the RPi case, the min is 1 and max is unbounded.\n> >\n> > I can see a possible bug(?) introduced in this series however. Say a\n> > pipeline\n> > handler sets maxQueueRequest to 16. An application comes along and\n> queues\n> > 20\n> > requests and then just waits for completion of those 20 requests. The\n> > PipelineHandler base class calls queueRequestDevice for the first 16\n> > requests,\n> > but nothing that I can see will trigger queuing of the last 4 requests\n> once\n> > the\n> > pipeline handler has completed the first N requests. Unless I've missed\n> > something?\n>\n> I think I saw Jacopo confirm the same in a parallel reply here, but I\n> understood this to be something we shouldn't expose to applications.\n> Applications can always choose to queue an unlimited number of requests\n> to a Camera (within the limits of the universe) ... but it's the\n> pipeline handlers responsibility to ensure that requests are not queued\n> beyond the capabilities of the IPA. As such - requests may stay 'queued'\n> in the pipeline handler or CameraData object until the IPA is ready to\n> process the next one.\n>\n\nI agree that the application never needs to know the maximum bound for\nrequests.\nBut I still see a problem where any pending requests queued\nin PipelineHandler::waitingRequests_\nwill never make it to PipelineHandlerXXX::queueRequestDevice() if the\napplication\ndoes not queue any further requests to the camera.\n\nRegards,\nNaush\n\n\n>\n> --\n> Kieran\n>\n>\n>\n> >\n> > Regards,\n> > Naush\n> >\n> >\n> > > Am I misunderstanding something?\n> > >\n> > >\n> > > Paul\n> > >\n> > > >\n> > > > >\n> > > > > I suppose then we'd actually need a compliance test for the\n> property,\n> > > > > but I guess we have a few more of those anyway.\n> > > > >\n> > > > > My two cents.\n> > > >\n> > > > Thanks ;)\n> > > > j\n> > > >\n> > > > >\n> > > > >\n> > > > > Paul\n> > > > >\n> > > > > > happen transparently if a pipeline handler register\n> > > > > > properties::PipelineDepth ? Am I overthinking this ?\n> > > > > >\n> > > > > >\n> > > > > > > >\n> > > > > > > > Sorry for not having been clear\n> > > > > > > >\n> > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > Also, can this feature be tested in test/ ? Probably\n> not, as\n> > > it\n> > > > > > > > > > requires a compliant pipeline handler ? Maybe vimc can be\n> > > instrumented\n> > > > > > > > > I have been thinking about that as well. It's shouldn't be\n> too\n> > > difficult.\n> > > > > > > > > > ?\n> > > > > > > > > >\n> > > > > > > > > > > The pipeline handler is now equipped with the\n> > > responsibility of not to\n> > > > > > > > > > > over queue the requests to the underlying layers. The\n> > > derived\n> > > > > > > > > > > pipeline handler (or even IPA) can also have various\n> kind\n> > > of requests\n> > > > > > > > > > > queue(arrays, queues or ring-buffer) hence, it might\n> be an\n> > > issue where\n> > > > > > > > > > > the application queues the requests at a rate where\n> these\n> > > kind of\n> > > > > > > > > > > queues can over-flow. The patch ensures that the base\n> > > PipelineHandler\n> > > > > > > > > > > will never let the requests overflow to underlying\n> layers,\n> > > once the\n> > > > > > > > > > > derived pipeline handler sets its maximum capacity of\n> > > handling\n> > > > > > > > > > > in-flight requests using\n> > > PipelineHandler::setMaxQueueRequests().\n> > > > > > > > > > >\n> > > > > > > > > > > The queue request management introduced in the pipeline\n> > > handler base\n> > > > > > > > > > > class is now used by the IPU3 and RkISP1 pipeline\n> > > handlers. This will\n> > > > > > > > > > > prevent over-writing of frame contexts (i.e. FCQueue)\n> in\n> > > these two\n> > > > > > > > > > > pipeline handlers.\n> > > > > > > > > > >\n> > > > > > > > > > > Signed-off-by: Umang Jain >\n> > > > > > > > > > > ---\n> > > > > > > > > > > include/libcamera/internal/pipeline_handler.h | 4\n> ++\n> > > > > > > > > > > src/libcamera/pipeline/ipu3/ipu3.cpp | 1 +\n> > > > > > > > > > > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +\n> > > > > > > > > > > src/libcamera/pipeline_handler.cpp | 51\n> > > ++++++++++++++++++-\n> > > > > > > > > > > 4 files changed, 56 insertions(+), 1 deletion(-)\n> > > > > > > > > > >\n> > > > > > > > > > > diff --git\n> a/include/libcamera/internal/pipeline_handler.h\n> > > b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > > index ec4f662d..83f8bd9f 100644\n> > > > > > > > > > > --- a/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > > +++ b/include/libcamera/internal/pipeline_handler.h\n> > > > > > > > > > > @@ -70,6 +70,7 @@ public:\n> > > > > > > > > > > protected:\n> > > > > > > > > > > void registerCamera(std::shared_ptr\n> > > camera);\n> > > > > > > > > > > void hotplugMediaDevice(MediaDevice *media);\n> > > > > > > > > > > + void setMaxQueueRequests(uint32_t maxRequests);\n> > > > > > > > > > >\n> > > > > > > > > > > virtual int queueRequestDevice(Camera *camera,\n> > > Request *request) = 0;\n> > > > > > > > > > > virtual void stopDevice(Camera *camera) = 0;\n> > > > > > > > > > > @@ -97,6 +98,9 @@ private:\n> > > > > > > > > > > Mutex lock_;\n> > > > > > > > > > > unsigned int useCount_\n> > > LIBCAMERA_TSA_GUARDED_BY(lock_);\n> > > > > > > > > > >\n> > > > > > > > > > > + uint32_t maxQueueRequests_;\n> > > > > > > > > > > + uint32_t requestsQueueCounter_;\n> > > > > > > > > > > +\n> > > > > > > > > > > friend class PipelineHandlerFactoryBase;\n> > > > > > > > > > > };\n> > > > > > > > > > >\n> > > > > > > > > > > diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > > index e4d79ea4..d1d42f78 100644\n> > > > > > > > > > > --- a/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > > +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp\n> > > > > > > > > > > @@ -422,6 +422,7 @@ CameraConfiguration::Status\n> > > IPU3CameraConfiguration::validate()\n> > > > > > > > > > >\n> PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager\n> > > *manager)\n> > > > > > > > > > > : PipelineHandler(manager),\n> > > cio2MediaDev_(nullptr), imguMediaDev_(nullptr)\n> > > > > > > > > > > {\n> > > > > > > > > > > +\n> setMaxQueueRequests(ipa::ipu3::kMaxFrameContexts);\n> > > > > > > > > > > }\n> > > > > > > > > > >\n> > > > > > > > > > > std::unique_ptr\n> > > > > > > > > > > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > > index eb9ad65c..a48adba9 100644\n> > > > > > > > > > > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > > > > > > > > > > @@ -599,6 +599,7 @@ CameraConfiguration::Status\n> > > RkISP1CameraConfiguration::validate()\n> > > > > > > > > > >\n> > > PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> > > > > > > > > > > : PipelineHandler(manager), hasSelfPath_(true)\n> > > > > > > > > > > {\n> > > > > > > > > > > +\n> > > setMaxQueueRequests(ipa::rkisp1::kMaxFrameContexts);\n> > > > > > > > > > > }\n> > > > > > > > > > >\n> > > > > > > > > > > std::unique_ptr\n> > > > > > > > > > > diff --git a/src/libcamera/pipeline_handler.cpp\n> > > b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > > index cfade490..103f9db0 100644\n> > > > > > > > > > > --- a/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > > +++ b/src/libcamera/pipeline_handler.cpp\n> > > > > > > > > > > @@ -67,7 +67,8 @@ LOG_DEFINE_CATEGORY(Pipeline)\n> > > > > > > > > > > * through the PipelineHandlerFactoryBase::create()\n> > > function.\n> > > > > > > > > > > */\n> > > > > > > > > > > PipelineHandler::PipelineHandler(CameraManager\n> *manager)\n> > > > > > > > > > > - : manager_(manager), useCount_(0)\n> > > > > > > > > > > + : manager_(manager), useCount_(0),\n> > > maxQueueRequests_(0),\n> > > > > > > > > > > + requestsQueueCounter_(0)\n> > > > > > > > > > > {\n> > > > > > > > > > > }\n> > > > > > > > > > >\n> > > > > > > > > > > @@ -428,6 +429,9 @@ void\n> > > PipelineHandler::doQueueRequest(Request *request)\n> > > > > > > > > > > Camera::Private *data = camera->_d();\n> > > > > > > > > > > data->queuedRequests_.push_back(request);\n> > > > > > > > > > >\n> > > > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > > > + requestsQueueCounter_++;\n> > > > > > > > > > > +\n> > > > > > > > > > > request->_d()->sequence_ =\n> > > data->requestSequence_++;\n> > > > > > > > > > >\n> > > > > > > > > > > if (request->_d()->cancelled_) {\n> > > > > > > > > > > @@ -455,6 +459,10 @@ void\n> > > PipelineHandler::doQueueRequests()\n> > > > > > > > > > > if (!request->_d()->prepared_)\n> > > > > > > > > > > break;\n> > > > > > > > > > >\n> > > > > > > > > > > + if (maxQueueRequests_ &&\n> > > > > > > > > > > + requestsQueueCounter_ >=\n> > > maxQueueRequests_)\n> > > > > > > > > > > + break;\n> > > > > > > > > > > +\n> > > > > > > > > > > doQueueRequest(request);\n> > > > > > > > > > > waitingRequests_.pop();\n> > > > > > > > > > > }\n> > > > > > > > > > > @@ -531,6 +539,9 @@ void\n> > > PipelineHandler::completeRequest(Request *request)\n> > > > > > > > > > > ASSERT(!req->hasPendingBuffers());\n> > > > > > > > > > > data->queuedRequests_.pop_front();\n> > > > > > > > > > > camera->requestComplete(req);\n> > > > > > > > > > > +\n> > > > > > > > > > > + if (maxQueueRequests_)\n> > > > > > > > > > > + requestsQueueCounter_--;\n> > > > > > > > > > > }\n> > > > > > > > > > > }\n> > > > > > > > > > >\n> > > > > > > > > > > @@ -647,6 +658,44 @@ void PipelineHandler::disconnect()\n> > > > > > > > > > > * constant for the whole lifetime of the pipeline\n> > > handler.\n> > > > > > > > > > > */\n> > > > > > > > > > >\n> > > > > > > > > > > +/**\n> > > > > > > > > > > + * \\var PipelineHandler::maxQueueRequests_\n> > > > > > > > > > > + * \\brief Maximum number of in-flight requests that\n> can\n> > > be queued\n> > > > > > > > > > > + *\n> > > > > > > > > > > + * A hardware can handle a certain number of maximum\n> > > requests at a given\n> > > > > > > > > > > + * point. If such a constraint exists, set\n> > > maxQueueRequests_ via\n> > > > > > > > > > > + * \\a setMaxQueueRequests() in the derived pipeline\n> > > handler.\n> > > > > > > > > > > + *\n> > > > > > > > > > > + * The derived pipeline handler can choose not to\n> define\n> > > such constraint as\n> > > > > > > > > > > + * well. In that case, the derived pipeline handler\n> can\n> > > avoid setting\n> > > > > > > > > > > + * \\a setMaxQueueReqeuests(), hence \\a\n> maxQueueRequests_\n> > > and\n> > > > > > > > > > > + * \\a requestsQueueCounter_ will be 0.\n> > > > > > > > > > > + */\n> > > > > > > > > > > +\n> > > > > > > > > > > +/**\n> > > > > > > > > > > + * \\var PipelineHandler::requestsQueueCounter_\n> > > > > > > > > > > + * \\brief Number of requests queued to the underlying\n> > > hardware\n> > > > > > > > > > > + *\n> > > > > > > > > > > + * If \\a setMaxQueueRequests() is set by the derived\n> > > pipeline handler,\n> > > > > > > > > > > + * requestsQueueCounter_ reflects the number of\n> requests\n> > > queued\n> > > > > > > > > > > + * to the underlying hardware by the pipeline handler.\n> > > > > > > > > > > + */\n> > > > > > > > > > > +\n> > > > > > > > > > > +/**\n> > > > > > > > > > > + * \\brief Sets the maximum number of requests that\n> can be\n> > > queued\n> > > > > > > > > > > + * \\param[in] maxRequests Maximum number of in-flight\n> > > requests\n> > > > > > > > > > > + *\n> > > > > > > > > > > + * A hardware can handle a certain number of requests\n> at\n> > > a given point.\n> > > > > > > > > > > + * This function sets the maximum number of in-flight\n> > > requests that can\n> > > > > > > > > > > + * be queued to the hardware by the pipeline handler.\n> > > Each derived pipeline\n> > > > > > > > > > > + * handler should set the maximum number of in-flight\n> > > requests it can handle\n> > > > > > > > > > > + * at a given point using this function, if at all\n> such a\n> > > constraint exists.\n> > > > > > > > > > > + */\n> > > > > > > > > > > +void PipelineHandler::setMaxQueueRequests(uint32_t\n> > > maxRequests)\n> > > > > > > > > > > +{\n> > > > > > > > > > > + maxQueueRequests_ = maxRequests;\n> > > > > > > > > > > +}\n> > > > > > > > > > > +\n> > > > > > > > > > > /**\n> > > > > > > > > > > * \\fn PipelineHandler::name()\n> > > > > > > > > > > * \\brief Retrieve the pipeline handler name\n> > > > > > > > > > > --\n> > > > > > > > > > > 2.38.1\n> > > > > > > > > > >\n> > > > > > >\n> > >\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 62D93C3200\n\tfor ;\n\tThu, 15 Dec 2022 10:00:10 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B106E60410;\n\tThu, 15 Dec 2022 11:00:09 +0100 (CET)","from mail-vk1-xa34.google.com (mail-vk1-xa34.google.com\n\t[IPv6:2607:f8b0:4864:20::a34])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B839A60410\n\tfor ;\n\tThu, 15 Dec 2022 11:00:07 +0100 (CET)","by mail-vk1-xa34.google.com with SMTP id l17so4466592vkk.3\n\tfor ;\n\tThu, 15 Dec 2022 02:00:07 -0800 (PST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671098409;\n\tbh=UhWwcTQGa36Mdf639mpi6TRvRn5Vi8SuCsYl09eaeXc=;\n\th=References:In-Reply-To:Date:To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=JRqcjcKS3mfLZNMxrvBgiq0RSj7OkbJJxG7GccgpPQeyrskARRsRLtEVda4pzRV7d\n\tgfhZL0FcUHlerO/QpGSHj8gKixVHkZ1I/62m7/l8iNopcL29hc7pOvFaXuR2+Thhfu\n\tSHef+oV6qwj1tWaY2+PmfRYAFO9stUyywLQrCGbzTo+PmU2p7X7EyDCooDwUxAk7mA\n\tSjyGJ6+7DTUjBOe8O4gBW1rjpHjP7V+mPUI3qY5E6i1wuqnb/rYH3exlIE7dyoNTQM\n\tQRIxcEauGW0bsiZ1j/Sgxw2YW9hnALnx2otbapt3qkw9lkRgfcZhZ5xiJRTUdsoMFC\n\teBOqlOnUcVFwg==","v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=kwsq1NnEo8ANBv88EnwrPe6gP39cBMp7vs37hkFd+8w=;\n\tb=i+LvP28LPDOfClNCsecMzhmVMKUVwkVA8A8HtQsibyTOkIqiWliXBK9gkHxZjBmIhX\n\tY2ozOPJBl/9RJcCd+pKrKZG4m2ctSblbW023L16P2yW24QzXjRlCEQpGyb24QIx5FCu6\n\tMgzFSwhPYt8NObEB7Ko75wg6SowdviRZxNJaG3DwKmSOJnVAOjjqT1AR82wQmKlKYxEU\n\tKm5G1BGlShKcE36D/tgLOh2hEZHyROgwPbkuG6ZLKuG1ohXXicnRW2zCua1Gxk3vrvGO\n\twyCAwvId9zYCv4g5vjUGEl6iXIyHIWuOBesIXxKzqVRlX8NdPYjqMJ0kT6X5Obrx8vDT\n\twqYg=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=raspberrypi.com\n\theader.i=@raspberrypi.com\n\theader.b=\"i+LvP28L\"; dkim-atps=neutral","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=kwsq1NnEo8ANBv88EnwrPe6gP39cBMp7vs37hkFd+8w=;\n\tb=KGJJVedxoPP5oSuc4oE008ijJiY3qUjgTLa4LZo+Z+ySyriADqNvCNG5r9ZKChPzqF\n\tZbFb5Q++E2HYq8CkE26O12aby+42KG8KQ8KL56UNetG9l/OKGk0zoOMnlEKD6nZmv8XP\n\tSmYUo0p04ufCnh1usj72h88QcicAJrMERJ1oCp4Cymflepa9/AouZ5nN9JO2Wmh2mKvm\n\t7kToxSxZ0IF6ETQ6lKn2uSLqWDvXabm44WQ/EuVLD7hSDocjM9TqXgL1XdKMD6676Pdz\n\t9mOI2PtC4JjnWPTxlN5D/xALynYfXlN7+N6ZXlGT9cckN2Fms1bwnSzQI4DLAHg1srx9\n\tSPTg==","X-Gm-Message-State":"ANoB5pm19pI4WUqpDpMLTI5uTOlc+S1KtmoCsiY/aAWeU43fHTlSqnrL\n\ttgzB4TUlQbdHGfpWAIYV9TbmBLGH+YdGABSBN+thQA==","X-Google-Smtp-Source":"AA0mqf6LpSiXMoIVdDEG/5V3KgU3mbgPHyD9EG4ZengXi6t1GvFMoJbQstgcD8pMoHWUz0wNmCggk/er36nr5k1haD4=","X-Received":"by 2002:a05:6122:d8c:b0:3bd:fe9b:c10b with SMTP id\n\tbc12-20020a0561220d8c00b003bdfe9bc10bmr3859068vkb.19.1671098406449;\n\tThu, 15 Dec 2022 02:00:06 -0800 (PST)","MIME-Version":"1.0","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221213155507.t2v5muvjw7y3tywr@uno.localdomain>\n\t<25266006-5dc3-ced2-8b69-743edc1a1f1f@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t\n\t\n\t<167109791908.9133.17756312340732843792@Monstersaurus>","In-Reply-To":"<167109791908.9133.17756312340732843792@Monstersaurus>","Date":"Thu, 15 Dec 2022 09:59:51 +0000","Message-ID":"","To":"Kieran Bingham ","Content-Type":"multipart/alternative; boundary=\"000000000000e6c9b005efdaea4a\"","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Naushir Patuck via libcamera-devel\n\t","Reply-To":"Naushir Patuck ","Cc":"Naushir Patuck via libcamera-devel ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26089,"web_url":"https://patchwork.libcamera.org/comment/26089/","msgid":"<167110131188.578223.13306471563427333560@Monstersaurus>","date":"2022-12-15T10:48:31","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Naushir Patuck (2022-12-15 09:59:51)\n> On Thu, 15 Dec 2022 at 09:52, Kieran Bingham <\n> kieran.bingham@ideasonboard.com> wrote:\n> \n> > Quoting Naushir Patuck via libcamera-devel (2022-12-15 09:43:55)\n> > > Hi all,\n> > >\n> > > On Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\n> > > libcamera-devel@lists.libcamera.org> wrote:\n> > >\n> > > > Hi Jacopo,\n> > > >\n> > > > On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n> > > > > Hi Paul\n> > > > >\n> > > > > On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n> > > > > > Hi,\n> > > > > >\n> > > > > > On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n> > > > libcamera-devel wrote:\n> > > > > > > Hi Umang\n> > > > > > >\n> > > > > > > On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n> > > > > > > > Hi Jacopo\n> > > > > > > >\n> > > > > > > > On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n> > > > > > > > > Hi Umang\n> > > > > > > > >\n> > > > > > > > > On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n> > > > > > > > > > Hi Jacopo,\n> > > > > > > > > >\n> > > > > > > > > > On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n> > > > > > > > > > > Hi Umang,\n> > > > > > > > > > >\n> > > > > > > > > > > On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n> > > > libcamera-devel wrote:\n> > > > > > > > > > > > This patch introduces a new constraint in the pipeline\n> > > > handler base\n> > > > > > > > > > > > class which allows various derived pipeline handlers to\n> > > > set and\n> > > > > > > > > > > > restrict the number of maximum in-flight requests that\n> > can\n> > > > be queued to\n> > > > > > > > > > > > the underlying components (for e.g. the IPA).\n> > > > > > > > > > > It's my understanding that the max number of in-flight\n> > > > requests is\n> > > > > > > > > > > what the properties::PipelineDepth property expresses.\n> > > > > > > > > > Indeed, that's why understanding as well. The\n> > > > properties::PipelineDepth\n> > > > > > > > > > should be set by the pipeline-handlers.\n> > > > > >\n> > > > > > (Besides the fact that I can't find properties::PipelineDepth)\n> > > > > > controls::PipelineDepth says that it \"Specifies the number of\n> > pipeline\n> > > > > > stages the frame went through from when it was exposed to when the\n> > > > final\n> > > > > > completed result was available to the framework.\", which to me\n> > sounds\n> > > > > > like the *minimum* amount of in-flight requests.\n> > > > >\n> > > > > It's a draft control copied in to please Android.\n> > > > > My understanding was that it had to be made a non-mutable property\n> > > >\n> > > > Ah, indeed it says that :)\n> > > >\n> > > > >\n> > > > > >\n> > > > > > I thought that here we're defining the *maximum* allowed in-flight\n> > > > > > requests. Not that all of the requests have to actually be doing\n> > > > > > something in hardware, but that it's the amount of requests the the\n> > > > > > pipeline handler is capable of handling internally without\n> > overflowing.\n> > > > > >\n> > > > > > > > > >\n> > > > > > > > > > We need to identify and atleast agree upon the correct\n> > > > PipelineDepth for the\n> > > > > > > > > > two platforms that uses the FCQueue (IPU3 and RkISP1) as\n> > well.\n> > > > Some grey\n> > > > > > > > > > areas there as well\n> > > > > > > > > I think 16 is fine for now. At least doesn't change what we\n> > > > already\n> > > > > > > > > have today\n> > > > > > > > >\n> > > > > > > > > > > Can't we access that property at queue request time, or\n> > > > maybe better we\n> > > > > > > > > > > could cache it at camera registration time, since it's\n> > > > constant ?\n> > > > > > > > > > If we intent to use the properties::PipelineDepth, as the\n> > size\n> > > > of the\n> > > > > > > > > > FCQueue - it needs to exist / set before IPA is\n> > constructed.\n> > > > > > > > > >\n> > > > > > > > > > (See IPAIPU3::IPAIPU3() for context)\n> > > > > > > > > >\n> > > > > > > > > The property can still be defined in the IPA header as IPA\n> > can\n> > > > access\n> > > > > > > > > it safely.\n> > > > > > > >\n> > > > > > > > You mean via a global const variable like it's done in Patch\n> > 1/2 ?\n> > > > > > >\n> > > > > > > Yes, like in patch 1/2\n> > > > > > >\n> > > > > > >\n> > > > > > > > >\n> > > > > > > > > Pipeline Handlers will have to register the property\n> > -before- the\n> > > > > > > > > camera is registered, but looking at IPU3 that's already the\n> > > > case now\n> > > > > > > > > (data->properties_ gets populated before calling\n> > > > registerCamera())\n> > > > > > > > >\n> > > > > > > > >\n> > > > > > > > > > So I think queue request time is quite late there. Camera\n> > > > registeration time\n> > > > > > > > > > might be possible with some changes (looking at\n> > > > > > > > > > PipelineHandlerIPU3::registerCameras())\n> > > > > > > > > What I meant is either\n> > > > > > > > >\n> > > > > > > > > 1) At PipelineHandler::queueRequest() time, the\n> > PipelineHandler\n> > > > base\n> > > > > > > > > class accesses Camera::properties() and fetches\n> > > > > > > > > properties::PipelineDepth\n> > > > > > > >\n> > > > > > > > I think it's a over-kill here. queueRequest() is a recurring\n> > call\n> > > > and I\n> > > > > > > > don't want fetching of PipelineDepth here.\n> > > > > >\n> > > > > > Oh one hand, yeah imo getting it from Camera::properties() is more\n> > > > > > correct, but on the other hand it does indeed seem overkill.\n> > > > > >\n> > > > > > >\n> > > > > > > While I consider a lookup on an unordered_map<> of a few items\n> > not\n> > > > > > > that expensive, I agree it's in an hot path, and it would be\n> > better\n> > > > to\n> > > > > > > cache the value somewhere. Caching it inside Camera or\n> > > > Camera::Private\n> > > > > > > is a duplication, as it's already part of\n> > > > > > > Camera::Private::properties_, but I would be fine with the\n> > > > > > > PipelineHandler base class doing something like this (as\n> > proposed in\n> > > > > > > my previous 2) point below)\n> > > > > > >\n> > > > > > > void PipelineHandler::registerCamera(std::shared_ptr\n> > camera)\n> > > > > > > {\n> > > > > > >\n> > > > > > > /* Cache the pipeline depth in the Camera private data. */\n> > > > > > > const auto &pipelineDepth =\n> > > > camera->properties().get(properties::PipelineDepth);\n> > > > > > >\n> > > > > > > Camera::Private *data = camera->_d();\n> > > > > > > data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n> > > > > > >\n> > > > > > > ...\n> > > > > > > }\n> > > > > > >\n> > > > > > > Otherwise the PipelineHandler base class can be added with a\n> > class\n> > > > > > > member, and each registred camera will overwrite that value, it's\n> > > > less\n> > > > > > > nice, but all cameras from the same PH instance will have the\n> > same\n> > > > > > > pipeline depth, so...\n> > > > > >\n> > > > > > Perhaps caching would be a good middle ground.\n> > > > > >\n> > > > > > >\n> > > > > > >\n> > > > > > > > > 2) At PipelineHandler::registerCamera() time the property\n> > value\n> > > > is\n> > > > > > > > > cached, but this would require to cache per-camera data in\n> > > > > > > > > PipelineHandler, something we don't have, or to populate a\n> > class\n> > > > > > > > > member of Camera::Private for that purpose..\n> > > > > > > >\n> > > > > > > > That's interesting because I initially didn't think of multiple\n> > > > cameras\n> > > > > > > > registered with one instance of pipeline-handler.\n> > > > > > > > It also makes me wonder -\n> > > > > > > >\n> > > > > > > > - Is properties::PipelineDepth Camera specific or\n> > > > platform-specific ? Seems\n> > > > > > > > now it's Camera specific\n> > > > > > > > - Then defining it IPA headers won't be a good option, in cases\n> > > > where\n> > > > > > > > PipelineDepth for different cameras differ.\n> > > > > > >\n> > > > > > > Why camera specific ? Each camera will indeed register it as a\n> > > > > > > property for applications to retrieve it, but the\n> > > > > > > value depends on the platform afaict and it can be a compile-time\n> > > > constant\n> > > > > > > defined by the IPA headers. Even better, the same compile-time\n> > > > > > > constant will be used to size the FCQ.\n> > > > > >\n> > > > > > If it's platform-specific then wouldn't the pipeline handler have\n> > to\n> > > > > > report it based on what platform it's running on? So we can't\n> > really\n> > > > > >hardcode a constant. Maybe putting it in properties is indeed\n> > better.\n> > > > >\n> > > > > As far as I get it, this property applies to pipelines with an ISP\n> > (no\n> > > > > uvc, no simple) and those pipelines by definition runs on a single\n> > > > > platform, don't they ?\n> > > >\n> > > > Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n> > > > but out-of-tree kernel patches)\n> > > >\n> > > > >\n> > > > > >\n> > > > > > >\n> > > > > > > >\n> > > > > > > > Need some thinking ...\n> > > > > > > >\n> > > > > > > > But none the less, this is a something that can be done on top\n> > > > with wider\n> > > > > > > > discussion / context on PipelineDepth. It shouldn't necessarily\n> > > > block the\n> > > > > > > > series.\n> > > > > > >\n> > > > > > > My whole point is that I think we should find a better mechanism\n> > for\n> > > > > > > registering pipeline depth compared to having pipeline handlers\n> > call\n> > > > > > > setMaxQueueRequests() as it requires an explicit action that can\n> > > > > > > easily be overlooked, while we want the queue overflow\n> > protection to\n> > > > > >\n> > > > > > I mean, we could initialize kMaxFrameContexts to an invalid value\n> > (-1?)\n> > > > > > and then force *all* pipeline handlers to explictly set it, even\n> > if to\n> > > > > > zero. Then it becomes really easy to catch.\n> > > > >\n> > > > > The property doesn't make much sense for UVC, right ?\n> > > > >\n> > > > > Not all pipelines will use libipa and have an FCQ (RPi in example at\n> > > > > the moment). I don't think forcing all of them is actually necessary.\n> > > >\n> > > > Hm, that's true. No initial invalid value then.\n> > > >\n> > > > >\n> > > > > But the pipelines that do need to throttle the number of requests in\n> > > > > flight with the help of the PipelineHandler base class should ideally\n> > > > > get it for free by simply registering the property ?\n> > > >\n> > > > Yeah that does sound more nice.\n> > > >\n> > > > >\n> > > > > >\n> > > > > > Or note it down as a compliancy thing and have lc-compliance test\n> > for\n> > > > > > it. I presume this will become a required property to be exposed\n> > > > anyway.\n> > > > > >\n> > > > > > In any case, the goal is to report to the user the \"maximum number\n> > of\n> > > > > > requests that are allowed to be queued in the camera\", right? So\n> > it'll\n> > > > > > have to go through properties anyway. Perhaps then it's best to\n> > expose\n> > > > > > it via properties (as it's already the interface we have for\n> > exposing\n> > > > > > properties to the user), and then the base PipelineHandler can\n> > cache it\n> > > > > > for the queuing guard. That would get rid of the special explicit\n> > call,\n> > > > > > and would still achieve the goal.\n> > > > >\n> > > > > That would be my preference. There might be a discussions if\n> > > > > properties::PipelineDepth is the right property to report it or it\n> > > > > will conflict with Android's pipeline depth, which is different.\n> > > >\n> > > > +1\n> > > >\n> > > > Although to me this sounds separate from PipelineDepth, because my\n> > > > understanding is that that is the maximum amount of Requests that can\n> > be\n> > > > in-flight, while what we want here is that plus some extra that the\n> > > > application is allowed to \"overqueue\" that the pipeline handler will\n> > > > automagically handle internally.\n> > > >\n> > >\n> > > My understanding of PipelineDepth is in-line with the above - it gives a\n> > > minimum\n> > > number of in-flight requests needed for the pipeline handler. This new\n> > > property must give an upper bound on the number of in-flight requests the\n> > > pipeline\n> > > handler can handle. For the RPi case, the min is 1 and max is unbounded.\n> > >\n> > > I can see a possible bug(?) introduced in this series however. Say a\n> > > pipeline\n> > > handler sets maxQueueRequest to 16. An application comes along and\n> > queues\n> > > 20\n> > > requests and then just waits for completion of those 20 requests. The\n> > > PipelineHandler base class calls queueRequestDevice for the first 16\n> > > requests,\n> > > but nothing that I can see will trigger queuing of the last 4 requests\n> > once\n> > > the\n> > > pipeline handler has completed the first N requests. Unless I've missed\n> > > something?\n> >\n> > I think I saw Jacopo confirm the same in a parallel reply here, but I\n> > understood this to be something we shouldn't expose to applications.\n> > Applications can always choose to queue an unlimited number of requests\n> > to a Camera (within the limits of the universe) ... but it's the\n> > pipeline handlers responsibility to ensure that requests are not queued\n> > beyond the capabilities of the IPA. As such - requests may stay 'queued'\n> > in the pipeline handler or CameraData object until the IPA is ready to\n> > process the next one.\n> >\n> \n> I agree that the application never needs to know the maximum bound for\n> requests.\n> But I still see a problem where any pending requests queued\n> in PipelineHandler::waitingRequests_\n> will never make it to PipelineHandlerXXX::queueRequestDevice() if the\n> application\n> does not queue any further requests to the camera.\n\nOh - I see what you mean, yes - if we only process the queue when\nrequests are 'queued' then this could occur. We'll have to make sure we\nprod the queue when a request completes too, which makes sense, as that\nis the event that free's up the opportunity to process the next request.\n\n--\nKieran","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id B9F3EC31E9\n\tfor ;\n\tThu, 15 Dec 2022 10:48:36 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id DB5396339D;\n\tThu, 15 Dec 2022 11:48:35 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D4CF760410\n\tfor ;\n\tThu, 15 Dec 2022 11:48:34 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 40A24327;\n\tThu, 15 Dec 2022 11:48:34 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671101315;\n\tbh=laul4FuYD4720VNFHvJmuHofCNI5L7D8zwmH/4iz4ag=;\n\th=In-Reply-To:References:To:Date:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=NPHup85VQCjm47IZaUOlp1BoSW2vRzMitPv5gfM07T4iggSc2Py5XQVcgbaACP+8S\n\tJr3Ue4SALIS+CUMu75b9ZhnHJFA5VRR8l86YxNx99Le7L349xjo3n3t+MYjI8xW01k\n\tBo27F4vXySXPzTWQ9mAgtYCB5XofwDxEdn8vqSKrd7XKbQtEEu5XDzuCoCJmeuVAfG\n\tdRiDbw4fxXn8td4NTHwLKswCJnwhb/9tiSF7wg8K1ahp8eR/4nJUy4V4yY/0hGcCRN\n\tWkAyqhON6sIZ0BDsnMwlm8dUK3zABHtkeugdxkPGJBVr0j4eroaX9CCEBCwJh0mUCZ\n\trHY1VGPJfpieg==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671101314;\n\tbh=laul4FuYD4720VNFHvJmuHofCNI5L7D8zwmH/4iz4ag=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=GWBNCiqJ32wxR5dGAbRaDbgHp4eOkAI53oCEh/roc2h73XztZtSUR9oxdVsiq00PN\n\tlhAHEmUsf1jL6yosWomsnTGRjesq55HAJhQ3AzOTuIUU0Z8QW2PS62ycNr/bmWsKWO\n\t1gBgkbWYve9AHq3pJ6MU22TQfI+TK4vwaOYRQ1QM="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"GWBNCiqJ\"; dkim-atps=neutral","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t\n\t\n\t<167109791908.9133.17756312340732843792@Monstersaurus>\n\t","To":"Naushir Patuck ","Date":"Thu, 15 Dec 2022 10:48:31 +0000","Message-ID":"<167110131188.578223.13306471563427333560@Monstersaurus>","User-Agent":"alot/0.10","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Kieran Bingham via libcamera-devel\n\t","Reply-To":"Kieran Bingham ","Cc":"Naushir Patuck via libcamera-devel ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26090,"web_url":"https://patchwork.libcamera.org/comment/26090/","msgid":"<50b5cfa9-b226-040e-b878-8f92a32c4a68@ideasonboard.com>","date":"2022-12-15T16:04:51","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi Kieran, Naushir,\n\nOn 12/15/22 4:18 PM, Kieran Bingham via libcamera-devel wrote:\n> Quoting Naushir Patuck (2022-12-15 09:59:51)\n>> On Thu, 15 Dec 2022 at 09:52, Kieran Bingham <\n>> kieran.bingham@ideasonboard.com> wrote:\n>>\n>>> Quoting Naushir Patuck via libcamera-devel (2022-12-15 09:43:55)\n>>>> Hi all,\n>>>>\n>>>> On Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\n>>>> libcamera-devel@lists.libcamera.org> wrote:\n>>>>\n>>>>> Hi Jacopo,\n>>>>>\n>>>>> On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n>>>>>> Hi Paul\n>>>>>>\n>>>>>> On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n>>>>>>> Hi,\n>>>>>>>\n>>>>>>> On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n>>>>> libcamera-devel wrote:\n>>>>>>>> Hi Umang\n>>>>>>>>\n>>>>>>>> On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n>>>>>>>>> Hi Jacopo\n>>>>>>>>>\n>>>>>>>>> On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n>>>>>>>>>> Hi Umang\n>>>>>>>>>>\n>>>>>>>>>> On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n>>>>>>>>>>> Hi Jacopo,\n>>>>>>>>>>>\n>>>>>>>>>>> On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n>>>>>>>>>>>> Hi Umang,\n>>>>>>>>>>>>\n>>>>>>>>>>>> On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n>>>>> libcamera-devel wrote:\n>>>>>>>>>>>>> This patch introduces a new constraint in the pipeline\n>>>>> handler base\n>>>>>>>>>>>>> class which allows various derived pipeline handlers to\n>>>>> set and\n>>>>>>>>>>>>> restrict the number of maximum in-flight requests that\n>>> can\n>>>>> be queued to\n>>>>>>>>>>>>> the underlying components (for e.g. the IPA).\n>>>>>>>>>>>> It's my understanding that the max number of in-flight\n>>>>> requests is\n>>>>>>>>>>>> what the properties::PipelineDepth property expresses.\n>>>>>>>>>>> Indeed, that's why understanding as well. The\n>>>>> properties::PipelineDepth\n>>>>>>>>>>> should be set by the pipeline-handlers.\n>>>>>>> (Besides the fact that I can't find properties::PipelineDepth)\n>>>>>>> controls::PipelineDepth says that it \"Specifies the number of\n>>> pipeline\n>>>>>>> stages the frame went through from when it was exposed to when the\n>>>>> final\n>>>>>>> completed result was available to the framework.\", which to me\n>>> sounds\n>>>>>>> like the *minimum* amount of in-flight requests.\n>>>>>> It's a draft control copied in to please Android.\n>>>>>> My understanding was that it had to be made a non-mutable property\n>>>>> Ah, indeed it says that :)\n>>>>>\n>>>>>>> I thought that here we're defining the *maximum* allowed in-flight\n>>>>>>> requests. Not that all of the requests have to actually be doing\n>>>>>>> something in hardware, but that it's the amount of requests the the\n>>>>>>> pipeline handler is capable of handling internally without\n>>> overflowing.\n>>>>>>>>>>> We need to identify and atleast agree upon the correct\n>>>>> PipelineDepth for the\n>>>>>>>>>>> two platforms that uses the FCQueue (IPU3 and RkISP1) as\n>>> well.\n>>>>> Some grey\n>>>>>>>>>>> areas there as well\n>>>>>>>>>> I think 16 is fine for now. At least doesn't change what we\n>>>>> already\n>>>>>>>>>> have today\n>>>>>>>>>>\n>>>>>>>>>>>> Can't we access that property at queue request time, or\n>>>>> maybe better we\n>>>>>>>>>>>> could cache it at camera registration time, since it's\n>>>>> constant ?\n>>>>>>>>>>> If we intent to use the properties::PipelineDepth, as the\n>>> size\n>>>>> of the\n>>>>>>>>>>> FCQueue - it needs to exist / set before IPA is\n>>> constructed.\n>>>>>>>>>>> (See IPAIPU3::IPAIPU3() for context)\n>>>>>>>>>>>\n>>>>>>>>>> The property can still be defined in the IPA header as IPA\n>>> can\n>>>>> access\n>>>>>>>>>> it safely.\n>>>>>>>>> You mean via a global const variable like it's done in Patch\n>>> 1/2 ?\n>>>>>>>> Yes, like in patch 1/2\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>>> Pipeline Handlers will have to register the property\n>>> -before- the\n>>>>>>>>>> camera is registered, but looking at IPU3 that's already the\n>>>>> case now\n>>>>>>>>>> (data->properties_ gets populated before calling\n>>>>> registerCamera())\n>>>>>>>>>>\n>>>>>>>>>>> So I think queue request time is quite late there. Camera\n>>>>> registeration time\n>>>>>>>>>>> might be possible with some changes (looking at\n>>>>>>>>>>> PipelineHandlerIPU3::registerCameras())\n>>>>>>>>>> What I meant is either\n>>>>>>>>>>\n>>>>>>>>>> 1) At PipelineHandler::queueRequest() time, the\n>>> PipelineHandler\n>>>>> base\n>>>>>>>>>> class accesses Camera::properties() and fetches\n>>>>>>>>>> properties::PipelineDepth\n>>>>>>>>> I think it's a over-kill here. queueRequest() is a recurring\n>>> call\n>>>>> and I\n>>>>>>>>> don't want fetching of PipelineDepth here.\n>>>>>>> Oh one hand, yeah imo getting it from Camera::properties() is more\n>>>>>>> correct, but on the other hand it does indeed seem overkill.\n>>>>>>>\n>>>>>>>> While I consider a lookup on an unordered_map<> of a few items\n>>> not\n>>>>>>>> that expensive, I agree it's in an hot path, and it would be\n>>> better\n>>>>> to\n>>>>>>>> cache the value somewhere. Caching it inside Camera or\n>>>>> Camera::Private\n>>>>>>>> is a duplication, as it's already part of\n>>>>>>>> Camera::Private::properties_, but I would be fine with the\n>>>>>>>> PipelineHandler base class doing something like this (as\n>>> proposed in\n>>>>>>>> my previous 2) point below)\n>>>>>>>>\n>>>>>>>> void PipelineHandler::registerCamera(std::shared_ptr\n>>> camera)\n>>>>>>>> {\n>>>>>>>>\n>>>>>>>> /* Cache the pipeline depth in the Camera private data. */\n>>>>>>>> const auto &pipelineDepth =\n>>>>> camera->properties().get(properties::PipelineDepth);\n>>>>>>>> Camera::Private *data = camera->_d();\n>>>>>>>> data->pipelineDepth = pipelineDepth ? 0 : *pipelineDepth;\n>>>>>>>>\n>>>>>>>> ...\n>>>>>>>> }\n>>>>>>>>\n>>>>>>>> Otherwise the PipelineHandler base class can be added with a\n>>> class\n>>>>>>>> member, and each registred camera will overwrite that value, it's\n>>>>> less\n>>>>>>>> nice, but all cameras from the same PH instance will have the\n>>> same\n>>>>>>>> pipeline depth, so...\n>>>>>>> Perhaps caching would be a good middle ground.\n>>>>>>>\n>>>>>>>>\n>>>>>>>>>> 2) At PipelineHandler::registerCamera() time the property\n>>> value\n>>>>> is\n>>>>>>>>>> cached, but this would require to cache per-camera data in\n>>>>>>>>>> PipelineHandler, something we don't have, or to populate a\n>>> class\n>>>>>>>>>> member of Camera::Private for that purpose..\n>>>>>>>>> That's interesting because I initially didn't think of multiple\n>>>>> cameras\n>>>>>>>>> registered with one instance of pipeline-handler.\n>>>>>>>>> It also makes me wonder -\n>>>>>>>>>\n>>>>>>>>> - Is properties::PipelineDepth Camera specific or\n>>>>> platform-specific ? Seems\n>>>>>>>>> now it's Camera specific\n>>>>>>>>> - Then defining it IPA headers won't be a good option, in cases\n>>>>> where\n>>>>>>>>> PipelineDepth for different cameras differ.\n>>>>>>>> Why camera specific ? Each camera will indeed register it as a\n>>>>>>>> property for applications to retrieve it, but the\n>>>>>>>> value depends on the platform afaict and it can be a compile-time\n>>>>> constant\n>>>>>>>> defined by the IPA headers. Even better, the same compile-time\n>>>>>>>> constant will be used to size the FCQ.\n>>>>>>> If it's platform-specific then wouldn't the pipeline handler have\n>>> to\n>>>>>>> report it based on what platform it's running on? So we can't\n>>> really\n>>>>>>> hardcode a constant. Maybe putting it in properties is indeed\n>>> better.\n>>>>>> As far as I get it, this property applies to pipelines with an ISP\n>>> (no\n>>>>>> uvc, no simple) and those pipelines by definition runs on a single\n>>>>>> platform, don't they ?\n>>>>> Well, the rkisp1 pipeline runs on imx8mp too... (with a few on-the-list\n>>>>> but out-of-tree kernel patches)\n>>>>>\n>>>>>>>>> Need some thinking ...\n>>>>>>>>>\n>>>>>>>>> But none the less, this is a something that can be done on top\n>>>>> with wider\n>>>>>>>>> discussion / context on PipelineDepth. It shouldn't necessarily\n>>>>> block the\n>>>>>>>>> series.\n>>>>>>>> My whole point is that I think we should find a better mechanism\n>>> for\n>>>>>>>> registering pipeline depth compared to having pipeline handlers\n>>> call\n>>>>>>>> setMaxQueueRequests() as it requires an explicit action that can\n>>>>>>>> easily be overlooked, while we want the queue overflow\n>>> protection to\n>>>>>>> I mean, we could initialize kMaxFrameContexts to an invalid value\n>>> (-1?)\n>>>>>>> and then force *all* pipeline handlers to explictly set it, even\n>>> if to\n>>>>>>> zero. Then it becomes really easy to catch.\n>>>>>> The property doesn't make much sense for UVC, right ?\n>>>>>>\n>>>>>> Not all pipelines will use libipa and have an FCQ (RPi in example at\n>>>>>> the moment). I don't think forcing all of them is actually necessary.\n>>>>> Hm, that's true. No initial invalid value then.\n>>>>>\n>>>>>> But the pipelines that do need to throttle the number of requests in\n>>>>>> flight with the help of the PipelineHandler base class should ideally\n>>>>>> get it for free by simply registering the property ?\n>>>>> Yeah that does sound more nice.\n>>>>>\n>>>>>>> Or note it down as a compliancy thing and have lc-compliance test\n>>> for\n>>>>>>> it. I presume this will become a required property to be exposed\n>>>>> anyway.\n>>>>>>> In any case, the goal is to report to the user the \"maximum number\n>>> of\n>>>>>>> requests that are allowed to be queued in the camera\", right? So\n>>> it'll\n>>>>>>> have to go through properties anyway. Perhaps then it's best to\n>>> expose\n>>>>>>> it via properties (as it's already the interface we have for\n>>> exposing\n>>>>>>> properties to the user), and then the base PipelineHandler can\n>>> cache it\n>>>>>>> for the queuing guard. That would get rid of the special explicit\n>>> call,\n>>>>>>> and would still achieve the goal.\n>>>>>> That would be my preference. There might be a discussions if\n>>>>>> properties::PipelineDepth is the right property to report it or it\n>>>>>> will conflict with Android's pipeline depth, which is different.\n>>>>> +1\n>>>>>\n>>>>> Although to me this sounds separate from PipelineDepth, because my\n>>>>> understanding is that that is the maximum amount of Requests that can\n>>> be\n>>>>> in-flight, while what we want here is that plus some extra that the\n>>>>> application is allowed to \"overqueue\" that the pipeline handler will\n>>>>> automagically handle internally.\n>>>>>\n>>>> My understanding of PipelineDepth is in-line with the above - it gives a\n>>>> minimum\n>>>> number of in-flight requests needed for the pipeline handler. This new\n>>>> property must give an upper bound on the number of in-flight requests the\n>>>> pipeline\n>>>> handler can handle. For the RPi case, the min is 1 and max is unbounded.\n>>>>\n>>>> I can see a possible bug(?) introduced in this series however. Say a\n>>>> pipeline\n>>>> handler sets maxQueueRequest to 16. An application comes along and\n>>> queues\n>>>> 20\n>>>> requests and then just waits for completion of those 20 requests. The\n>>>> PipelineHandler base class calls queueRequestDevice for the first 16\n>>>> requests,\n>>>> but nothing that I can see will trigger queuing of the last 4 requests\n>>> once\n>>>> the\n>>>> pipeline handler has completed the first N requests. Unless I've missed\n>>>> something?\n>>> I think I saw Jacopo confirm the same in a parallel reply here, but I\n>>> understood this to be something we shouldn't expose to applications.\n>>> Applications can always choose to queue an unlimited number of requests\n>>> to a Camera (within the limits of the universe) ... but it's the\n\nAre we advertising this as a standard policy ? If yes, I want this more \nexplicit to be documented since I have had a hard time looking for it.\n\nThough the PipelineDepth is still under discussion, it was asssumed \n(till today?) that applications would likely queue requests within a \ncertain bound.\n>>> pipeline handlers responsibility to ensure that requests are not queued\n>>> beyond the capabilities of the IPA. As such - requests may stay 'queued'\n>>> in the pipeline handler or CameraData object until the IPA is ready to\n>>> process the next one.\n>>>\n>> I agree that the application never needs to know the maximum bound for\n>> requests.\n>> But I still see a problem where any pending requests queued\n>> in PipelineHandler::waitingRequests_\n>> will never make it to PipelineHandlerXXX::queueRequestDevice() if the\n>> application\n>> does not queue any further requests to the camera.\n\nThe queuing of 20 requests in one go - sounds like a special case to me. \nUsually, applications (atleast what I expected) queue a certain number \nof requests and wait for their completion (i.e. the requestCompleted \ncallback to invoke) and then queue more requests through the callback \nhandler.\n> Oh - I see what you mean, yes - if we only process the queue when\n> requests are 'queued' then this could occur. We'll have to make sure we\n> prod the queue when a request completes too, which makes sense, as that\n> is the event that free's up the opportunity to process the next request.\n\nI agree that this can happen. More so, when the pipeline depth (no. of \nprocessing blocks each frame) is huge.\n\nBut what are we really doing here - we are transferring the intended \nbehavior(as above) from application's requestCompleted callback to \npipeline-handler base class. So conceptually both the behaviour are \nsimilar in my opinion. However, stating/document the infinite queuing of \nrequests for application's PoV would be a nice addition on top and would \nmake the corresponding  patch much more obvious.\n\nApart from that, it would be nice to document -\n\n             Requests required to be queued to capture X frames\n\n  Since this is also something I didn't find explicit information on and \nit would possibly be dependent on pipeline-depth (no. of processing \nblocks) (?). There is a lc-compliance test regarding this - \"Test single \ncapture cycles\" that test X request for X captures but on the other \nhand, some tests get skipped with the following error:\n\n[7:43:48.210256888] [55043]  INFO Camera camera.cpp:1026 configuring \nstreams: (0) 640x480-MJPEG\nCamera needs 4 requests, can't test only 2\n../src/apps/lc-compliance/simple_capture.cpp:91: Skipped\n\nSo I suspect some grey area here as well ? Possibly I need to tinker on \nother platforms to really use how many requests it takes to capture X \nno. of frames ...\n>\n> --\n> Kieran","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id B3534C31E9\n\tfor ;\n\tThu, 15 Dec 2022 16:05:01 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id F3B5D6339C;\n\tThu, 15 Dec 2022 17:05:00 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 732636334D\n\tfor ;\n\tThu, 15 Dec 2022 17:04:59 +0100 (CET)","from [IPV6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d] (unknown\n\t[IPv6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 1B40E327;\n\tThu, 15 Dec 2022 17:04:56 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671120301;\n\tbh=1/UhisW1TP7vJE8Bg6C9uWYQ2479QhCuYnQ2GOg4MbY=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=xxVnrmoAt1jEzOyak7aIhsc1nAEfmX0cCCB1s7Thf08TmL+maWJG+CH6nF+g/77gn\n\tfpvGIOcx6Zs8uJDbX/WDnCMuv5dF3dSzNRnjbhHeaVHdmtcy4fDWolvtqyj/n8Tc5D\n\tSTAZuRJNcvHSmv8H8qr2QxwoVKbsFlo9A03apLODGEUV/vLLO4RpxgmvTgyWY+1LHS\n\tMNxNVRt1N8670U4O9M4HhQwPNr6i4qmKOIoJWhD38y58VNRSIetJyKTaDTERGNrv5r\n\tJUDyp8ve/u2LwDf8h5zJ/BWp5fK6s1o7hN4BopoJsgRH3VuPXuULI+dmO/VYytk8VC\n\t6YbLrUTGkhHBw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671120298;\n\tbh=1/UhisW1TP7vJE8Bg6C9uWYQ2479QhCuYnQ2GOg4MbY=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=bSbKjxxCIQ5M8GbIdel1iR1PP6ZgKNM7HjYv6GlGpW1Y0WxYlJ/bHY1eWd+KeGi4Z\n\tOed1Oj2RYrXQOFhpLmC6cH56q9J/SRESSyyLN7BA7ERIwg8XUZx3bia+4KpYoUE9L7\n\tmzUJzgMlHTYOdFQgwksmhuV7/mqb8R3EnrLdHHng="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"bSbKjxxC\"; dkim-atps=neutral","Message-ID":"<50b5cfa9-b226-040e-b878-8f92a32c4a68@ideasonboard.com>","Date":"Thu, 15 Dec 2022 21:34:51 +0530","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n\tThunderbird/102.5.1","Content-Language":"en-US","To":"Kieran Bingham ,\n\tNaushir Patuck ","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t\n\t\n\t<167109791908.9133.17756312340732843792@Monstersaurus>\n\t\n\t<167110131188.578223.13306471563427333560@Monstersaurus>","In-Reply-To":"<167110131188.578223.13306471563427333560@Monstersaurus>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"8bit","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Umang Jain via libcamera-devel ","Reply-To":"Umang Jain ","Cc":"Naushir Patuck via libcamera-devel ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":26091,"web_url":"https://patchwork.libcamera.org/comment/26091/","msgid":"<813df948-c384-86a1-03fb-2207f15964b0@ideasonboard.com>","date":"2022-12-15T16:16:05","subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi again,\n\none more comment on this front,\n\nOn 12/15/22 9:34 PM, Umang Jain via libcamera-devel wrote:\n> Hi Kieran, Naushir,\n>\n> On 12/15/22 4:18 PM, Kieran Bingham via libcamera-devel wrote:\n>> Quoting Naushir Patuck (2022-12-15 09:59:51)\n>>> On Thu, 15 Dec 2022 at 09:52, Kieran Bingham <\n>>> kieran.bingham@ideasonboard.com> wrote:\n>>>\n>>>> Quoting Naushir Patuck via libcamera-devel (2022-12-15 09:43:55)\n>>>>> Hi all,\n>>>>>\n>>>>> On Thu, 15 Dec 2022 at 05:18, Paul Elder via libcamera-devel <\n>>>>> libcamera-devel@lists.libcamera.org> wrote:\n>>>>>\n>>>>>> Hi Jacopo,\n>>>>>>\n>>>>>> On Wed, Dec 14, 2022 at 02:41:30PM +0100, Jacopo Mondi wrote:\n>>>>>>> Hi Paul\n>>>>>>>\n>>>>>>> On Wed, Dec 14, 2022 at 09:13:06PM +0900, Paul Elder wrote:\n>>>>>>>> Hi,\n>>>>>>>>\n>>>>>>>> On Wed, Dec 14, 2022 at 10:19:54AM +0100, Jacopo Mondi via\n>>>>>> libcamera-devel wrote:\n>>>>>>>>> Hi Umang\n>>>>>>>>>\n>>>>>>>>> On Wed, Dec 14, 2022 at 02:11:12PM +0530, Umang Jain wrote:\n>>>>>>>>>> Hi Jacopo\n>>>>>>>>>>\n>>>>>>>>>> On 12/14/22 1:29 PM, Jacopo Mondi wrote:\n>>>>>>>>>>> Hi Umang\n>>>>>>>>>>>\n>>>>>>>>>>> On Wed, Dec 14, 2022 at 11:07:25AM +0530, Umang Jain wrote:\n>>>>>>>>>>>> Hi Jacopo,\n>>>>>>>>>>>>\n>>>>>>>>>>>> On 12/13/22 9:25 PM, Jacopo Mondi wrote:\n>>>>>>>>>>>>> Hi Umang,\n>>>>>>>>>>>>>\n>>>>>>>>>>>>> On Mon, Dec 12, 2022 at 08:32:56PM +0530, Umang Jain via\n>>>>>> libcamera-devel wrote:\n>>>>>>>>>>>>>> This patch introduces a new constraint in the pipeline\n>>>>>> handler base\n>>>>>>>>>>>>>> class which allows various derived pipeline handlers to\n>>>>>> set and\n>>>>>>>>>>>>>> restrict the number of maximum in-flight requests that\n>>>> can\n>>>>>> be queued to\n>>>>>>>>>>>>>> the underlying components (for e.g. the IPA).\n>>>>>>>>>>>>> It's my understanding that the max number of in-flight\n>>>>>> requests is\n>>>>>>>>>>>>> what the properties::PipelineDepth property expresses.\n>>>>>>>>>>>> Indeed, that's why understanding as well. The\n>>>>>> properties::PipelineDepth\n>>>>>>>>>>>> should be set by the pipeline-handlers.\n>>>>>>>> (Besides the fact that I can't find properties::PipelineDepth)\n>>>>>>>> controls::PipelineDepth says that it \"Specifies the number of\n>>>> pipeline\n>>>>>>>> stages the frame went through from when it was exposed to when the\n>>>>>> final\n>>>>>>>> completed result was available to the framework.\", which to me\n>>>> sounds\n>>>>>>>> like the *minimum* amount of in-flight requests.\n>>>>>>> It's a draft control copied in to please Android.\n>>>>>>> My understanding was that it had to be made a non-mutable property\n>>>>>> Ah, indeed it says that :)\n>>>>>>\n>>>>>>>> I thought that here we're defining the *maximum* allowed in-flight\n>>>>>>>> requests. Not that all of the requests have to actually be doing\n>>>>>>>> something in hardware, but that it's the amount of requests the \n>>>>>>>> the\n>>>>>>>> pipeline handler is capable of handling internally without\n>>>> overflowing.\n>>>>>>>>>>>> We need to identify and atleast agree upon the correct\n>>>>>> PipelineDepth for the\n>>>>>>>>>>>> two platforms that uses the FCQueue (IPU3 and RkISP1) as\n>>>> well.\n>>>>>> Some grey\n>>>>>>>>>>>> areas there as well\n>>>>>>>>>>> I think 16 is fine for now. At least doesn't change what we\n>>>>>> already\n>>>>>>>>>>> have today\n>>>>>>>>>>>\n>>>>>>>>>>>>> Can't we access that property at queue request time, or\n>>>>>> maybe better we\n>>>>>>>>>>>>> could cache it at camera registration time, since it's\n>>>>>> constant ?\n>>>>>>>>>>>> If we intent to use the properties::PipelineDepth, as the\n>>>> size\n>>>>>> of the\n>>>>>>>>>>>> FCQueue - it needs to exist / set before IPA is\n>>>> constructed.\n>>>>>>>>>>>> (See IPAIPU3::IPAIPU3() for context)\n>>>>>>>>>>>>\n>>>>>>>>>>> The property can still be defined in the IPA header as IPA\n>>>> can\n>>>>>> access\n>>>>>>>>>>> it safely.\n>>>>>>>>>> You mean via a global const variable like it's done in Patch\n>>>> 1/2 ?\n>>>>>>>>> Yes, like in patch 1/2\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>>> Pipeline Handlers will have to register the property\n>>>> -before- the\n>>>>>>>>>>> camera is registered, but looking at IPU3 that's already the\n>>>>>> case now\n>>>>>>>>>>> (data->properties_ gets populated before calling\n>>>>>> registerCamera())\n>>>>>>>>>>>\n>>>>>>>>>>>> So I think queue request time is quite late there. Camera\n>>>>>> registeration time\n>>>>>>>>>>>> might be possible with some changes (looking at\n>>>>>>>>>>>> PipelineHandlerIPU3::registerCameras())\n>>>>>>>>>>> What I meant is either\n>>>>>>>>>>>\n>>>>>>>>>>> 1) At PipelineHandler::queueRequest() time, the\n>>>> PipelineHandler\n>>>>>> base\n>>>>>>>>>>> class accesses Camera::properties() and fetches\n>>>>>>>>>>> properties::PipelineDepth\n>>>>>>>>>> I think it's a over-kill here. queueRequest() is a recurring\n>>>> call\n>>>>>> and I\n>>>>>>>>>> don't want fetching of PipelineDepth here.\n>>>>>>>> Oh one hand, yeah imo getting it from Camera::properties() is more\n>>>>>>>> correct, but on the other hand it does indeed seem overkill.\n>>>>>>>>\n>>>>>>>>> While I consider a lookup on an unordered_map<> of a few items\n>>>> not\n>>>>>>>>> that expensive, I agree it's in an hot path, and it would be\n>>>> better\n>>>>>> to\n>>>>>>>>> cache the value somewhere. Caching it inside Camera or\n>>>>>> Camera::Private\n>>>>>>>>> is a duplication, as it's already part of\n>>>>>>>>> Camera::Private::properties_, but I would be fine with the\n>>>>>>>>> PipelineHandler base class doing something like this (as\n>>>> proposed in\n>>>>>>>>> my previous 2) point below)\n>>>>>>>>>\n>>>>>>>>> void PipelineHandler::registerCamera(std::shared_ptr\n>>>> camera)\n>>>>>>>>> {\n>>>>>>>>>\n>>>>>>>>>    /* Cache the pipeline depth in the Camera private data. */\n>>>>>>>>>          const auto &pipelineDepth =\n>>>>>> camera->properties().get(properties::PipelineDepth);\n>>>>>>>>>    Camera::Private *data = camera->_d();\n>>>>>>>>>          data->pipelineDepth = pipelineDepth ? 0 : \n>>>>>>>>> *pipelineDepth;\n>>>>>>>>>\n>>>>>>>>>          ...\n>>>>>>>>> }\n>>>>>>>>>\n>>>>>>>>> Otherwise the PipelineHandler base class can be added with a\n>>>> class\n>>>>>>>>> member, and each registred camera will overwrite that value, it's\n>>>>>> less\n>>>>>>>>> nice, but all cameras from the same PH instance will have the\n>>>> same\n>>>>>>>>> pipeline depth, so...\n>>>>>>>> Perhaps caching would be a good middle ground.\n>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>>> 2) At PipelineHandler::registerCamera() time the property\n>>>> value\n>>>>>> is\n>>>>>>>>>>> cached, but this would require to cache per-camera data in\n>>>>>>>>>>> PipelineHandler, something we don't have, or to populate a\n>>>> class\n>>>>>>>>>>> member of Camera::Private for that purpose..\n>>>>>>>>>> That's interesting because I initially didn't think of multiple\n>>>>>> cameras\n>>>>>>>>>> registered with one instance of pipeline-handler.\n>>>>>>>>>> It also makes me wonder -\n>>>>>>>>>>\n>>>>>>>>>> - Is properties::PipelineDepth Camera specific or\n>>>>>> platform-specific ? Seems\n>>>>>>>>>> now it's Camera specific\n>>>>>>>>>> - Then defining it IPA headers won't be a good option, in cases\n>>>>>> where\n>>>>>>>>>> PipelineDepth for different cameras differ.\n>>>>>>>>> Why camera specific ? Each camera will indeed register it as a\n>>>>>>>>> property for applications to retrieve it, but the\n>>>>>>>>> value depends on the platform afaict and it can be a compile-time\n>>>>>> constant\n>>>>>>>>> defined by the IPA headers. Even better, the same compile-time\n>>>>>>>>> constant will be used to size the FCQ.\n>>>>>>>> If it's platform-specific then wouldn't the pipeline handler have\n>>>> to\n>>>>>>>> report it based on what platform it's running on? So we can't\n>>>> really\n>>>>>>>> hardcode a constant. Maybe putting it in properties is indeed\n>>>> better.\n>>>>>>> As far as I get it, this property applies to pipelines with an ISP\n>>>> (no\n>>>>>>> uvc, no simple) and those pipelines by definition runs on a single\n>>>>>>> platform, don't they ?\n>>>>>> Well, the rkisp1 pipeline runs on imx8mp too... (with a few \n>>>>>> on-the-list\n>>>>>> but out-of-tree kernel patches)\n>>>>>>\n>>>>>>>>>> Need some thinking ...\n>>>>>>>>>>\n>>>>>>>>>> But none the less, this is a something that can be done on top\n>>>>>> with wider\n>>>>>>>>>> discussion / context on PipelineDepth. It shouldn't necessarily\n>>>>>> block the\n>>>>>>>>>> series.\n>>>>>>>>> My whole point is that I think we should find a better mechanism\n>>>> for\n>>>>>>>>> registering pipeline depth compared to having pipeline handlers\n>>>> call\n>>>>>>>>> setMaxQueueRequests() as it requires an explicit action that can\n>>>>>>>>> easily be overlooked, while we want the queue overflow\n>>>> protection to\n>>>>>>>> I mean, we could initialize kMaxFrameContexts to an invalid value\n>>>> (-1?)\n>>>>>>>> and then force *all* pipeline handlers to explictly set it, even\n>>>> if to\n>>>>>>>> zero. Then it becomes really easy to catch.\n>>>>>>> The property doesn't make much sense for UVC, right ?\n>>>>>>>\n>>>>>>> Not all pipelines will use libipa and have an FCQ (RPi in \n>>>>>>> example at\n>>>>>>> the moment). I don't think forcing all of them is actually \n>>>>>>> necessary.\n>>>>>> Hm, that's true. No initial invalid value then.\n>>>>>>\n>>>>>>> But the pipelines that do need to throttle the number of \n>>>>>>> requests in\n>>>>>>> flight with the help of the PipelineHandler base class should \n>>>>>>> ideally\n>>>>>>> get it for free by simply registering the property ?\n>>>>>> Yeah that does sound more nice.\n>>>>>>\n>>>>>>>> Or note it down as a compliancy thing and have lc-compliance test\n>>>> for\n>>>>>>>> it. I presume this will become a required property to be exposed\n>>>>>> anyway.\n>>>>>>>> In any case, the goal is to report to the user the \"maximum number\n>>>> of\n>>>>>>>> requests that are allowed to be queued in the camera\", right?  So\n>>>> it'll\n>>>>>>>> have to go through properties anyway. Perhaps then it's best to\n>>>> expose\n>>>>>>>> it via properties (as it's already the interface we have for\n>>>> exposing\n>>>>>>>> properties to the user), and then the base PipelineHandler can\n>>>> cache it\n>>>>>>>> for the queuing guard. That would get rid of the special explicit\n>>>> call,\n>>>>>>>> and would still achieve the goal.\n>>>>>>> That would be my preference. There might be a discussions if\n>>>>>>> properties::PipelineDepth is the right property to report it or it\n>>>>>>> will conflict with Android's pipeline depth, which is different.\n>>>>>> +1\n>>>>>>\n>>>>>> Although to me this sounds separate from PipelineDepth, because my\n>>>>>> understanding is that that is the maximum amount of Requests that \n>>>>>> can\n>>>> be\n>>>>>> in-flight, while what we want here is that plus some extra that the\n>>>>>> application is allowed to \"overqueue\" that the pipeline handler will\n>>>>>> automagically handle internally.\n>>>>>>\n>>>>> My understanding of PipelineDepth is in-line with the above - it \n>>>>> gives a\n>>>>> minimum\n>>>>> number of in-flight requests needed for the pipeline handler.  \n>>>>> This new\n>>>>> property must give an upper bound on the number of in-flight \n>>>>> requests the\n>>>>> pipeline\n>>>>> handler can handle.  For the RPi case, the min is 1 and max is \n>>>>> unbounded.\n>>>>>\n>>>>> I can see a possible bug(?) introduced in this series however.  Say a\n>>>>> pipeline\n>>>>> handler sets maxQueueRequest to 16.  An application comes along and\n>>>> queues\n>>>>> 20\n>>>>> requests and then just waits for completion of those 20 requests. The\n>>>>> PipelineHandler base class calls queueRequestDevice for the first 16\n>>>>> requests,\n>>>>> but nothing that I can see will trigger queuing of the last 4 \n>>>>> requests\n>>>> once\n>>>>> the\n>>>>> pipeline handler has completed the first N requests. Unless I've \n>>>>> missed\n>>>>> something?\n>>>> I think I saw Jacopo confirm the same in a parallel reply here, but I\n>>>> understood this to be something we shouldn't expose to applications.\n>>>> Applications can always choose to queue an unlimited number of \n>>>> requests\n>>>> to a Camera (within the limits of the universe) ... but it's the\n>\n> Are we advertising this as a standard policy ? If yes, I want this \n> more explicit to be documented since I have had a hard time looking \n> for it.\n>\n> Though the PipelineDepth is still under discussion, it was asssumed \n> (till today?) that applications would likely queue requests within a \n> certain bound.\n>>>> pipeline handlers responsibility to ensure that requests are not \n>>>> queued\n>>>> beyond the capabilities of the IPA. As such - requests may stay \n>>>> 'queued'\n>>>> in the pipeline handler or CameraData object until the IPA is ready to\n>>>> process the next one.\n>>>>\n>>> I agree that the application never needs to know the maximum bound for\n>>> requests.\n>>> But I still see a problem where any pending requests queued\n>>> in PipelineHandler::waitingRequests_\n>>> will never make it to PipelineHandlerXXX::queueRequestDevice() if the\n>>> application\n>>> does not queue any further requests to the camera.\n>\n> The queuing of 20 requests in one go - sounds like a special case to \n> me. Usually, applications (atleast what I expected) queue a certain \n> number of requests and wait for their completion (i.e. the \n> requestCompleted callback to invoke) and then queue more requests \n> through the callback handler.\n>> Oh - I see what you mean, yes - if we only process the queue when\n>> requests are 'queued' then this could occur. We'll have to make sure we\n\nIt's not actually upon queuing of request. The waitingRequests_ is \nprocessed when the request has finished 'preparing' i.e. \ndoQueueRequest() is a callback to Request::Private::prepared signal.\n\nSo it depends where/how to invoke the Request::Private::Prepare() - even \nafter you queue the Request.\n>> prod the queue when a request completes too, which makes sense, as that\n>> is the event that free's up the opportunity to process the next request.\n>\n> I agree that this can happen. More so, when the pipeline depth (no. of \n> processing blocks each frame) is huge.\n>\n> But what are we really doing here - we are transferring the intended \n> behavior(as above) from application's requestCompleted callback to \n> pipeline-handler base class. So conceptually both the behaviour are \n> similar in my opinion. However, stating/document the infinite queuing \n> of requests for application's PoV would be a nice addition on top and \n> would make the corresponding  patch much more obvious.\n>\n> Apart from that, it would be nice to document -\n>\n>             Requests required to be queued to capture X frames\n>\n>  Since this is also something I didn't find explicit information on \n> and it would possibly be dependent on pipeline-depth (no. of \n> processing blocks) (?). There is a lc-compliance test regarding this - \n> \"Test single capture cycles\" that test X request for X captures but on \n> the other hand, some tests get skipped with the following error:\n>\n> [7:43:48.210256888] [55043]  INFO Camera camera.cpp:1026 configuring \n> streams: (0) 640x480-MJPEG\n> Camera needs 4 requests, can't test only 2\n> ../src/apps/lc-compliance/simple_capture.cpp:91: Skipped\n>\n> So I suspect some grey area here as well ? Possibly I need to tinker \n> on other platforms to really use how many requests it takes to capture \n> X no. of frames ...\n>>\n>> -- \n>> Kieran\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E79E5C31E9\n\tfor ;\n\tThu, 15 Dec 2022 16:16:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 5A8CD6334D;\n\tThu, 15 Dec 2022 17:16:13 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D50296334D\n\tfor ;\n\tThu, 15 Dec 2022 17:16:11 +0100 (CET)","from [IPV6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d] (unknown\n\t[IPv6:2401:4900:1f3e:7d24:3f0:3e81:fb16:ab4d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C3D8E327;\n\tThu, 15 Dec 2022 17:16:09 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1671120973;\n\tbh=upH2ybQ6pmz83m2yETtdwToc+vK8WQ+hySED9djJOHg=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=3vmxrHzEHMQJcTojdHMJJwXljeUYDsD2RSHyKjpzWZx4cYk2/CPlwsicqeoVt3M4W\n\t0OHKalb/KA/xYd/hqY1jb5DV1jsujJOYgwhnMUEqRWoMD6gNHW01I3TW+0jUWHNIJY\n\t9RL9pb7Kt6fg3OBD/C78c3rlewOPksltEajyu3h7zH2zAO0Ym6wphf2GYlZwNFhYij\n\tLvXseRQoIIHGqPWr23pbdYXXp/QsCcGAKK2R/qUnLQZ2kPoHGFDaIAC44jXLqQoaT5\n\tOdyH+HDxCuQOcQsSAkh9edfVHR+AUbw5FMg4CPNEYwmBRcm3Z+9suIp7DLpqoa/0SN\n\tcTJjhNiwZmzcw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1671120971;\n\tbh=upH2ybQ6pmz83m2yETtdwToc+vK8WQ+hySED9djJOHg=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=jfP/e9mgt/rDr4nZUzDskNLbx5noKg3eBvUSmYc1k0XOEwfAP3zbW+VXIGHLRjkZV\n\tFwyMyg1PAVlI1hIvZGHwJhWVnTFqz+pUUWsJrqsld29ZoxqnwAEH3ldwnl8gm0a6gq\n\tA8k4BewDPzHq2YF+d+f1somKerDQ2KT97Jybfp2s="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"jfP/e9mg\"; dkim-atps=neutral","Message-ID":"<813df948-c384-86a1-03fb-2207f15964b0@ideasonboard.com>","Date":"Thu, 15 Dec 2022 21:46:05 +0530","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n\tThunderbird/102.5.1","Content-Language":"en-US","To":"Kieran Bingham ,\n\tNaushir Patuck ","References":"<20221212150256.69004-1-umang.jain@ideasonboard.com>\n\t<20221214075908.xsu4xxl375b47ztk@uno.localdomain>\n\t\n\t<20221214091954.igijxckzroa5buyl@uno.localdomain>\n\t\n\t<20221214134130.flgk2ztujuwctnei@uno.localdomain>\n\t\n\t\n\t<167109791908.9133.17756312340732843792@Monstersaurus>\n\t\n\t<167110131188.578223.13306471563427333560@Monstersaurus>\n\t<50b5cfa9-b226-040e-b878-8f92a32c4a68@ideasonboard.com>","In-Reply-To":"<50b5cfa9-b226-040e-b878-8f92a32c4a68@ideasonboard.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"8bit","Subject":"Re: [libcamera-devel] [PATCH 2/2] libcamera: pipeline-handler:\n\tConsider max in-flight requests constraint","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Umang Jain via libcamera-devel ","Reply-To":"Umang Jain ","Cc":"Naushir Patuck via libcamera-devel ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]