[{"id":23858,"web_url":"https://patchwork.libcamera.org/comment/23858/","msgid":"<CAPhyPA40aGa7fTrRWXRBuSecANVx1SzwvOUbaxLt6H_P=d8nbA@mail.gmail.com>","date":"2022-07-13T12:13:29","subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","submitter":{"id":130,"url":"https://patchwork.libcamera.org/api/people/130/","name":"Nick Hollinghurst","email":"nick.hollinghurst@raspberrypi.com"},"content":"Hi Naush,\n\nThanks - that seems to fix it\n\nOn Wed, 13 Jul 2022 at 13:00, Naushir Patuck <naush@raspberrypi.com> wrote:\n\n> The logic used to match asynchronous image and embedded buffers was being\n> overly\n> aggressive by possibly allowing an unmatched image buffer to be sent to\n> the IPA\n> if the matching embedded buffer had not yet been dequeued. This condition\n> only\n> occurs when the system is heavily loaded and dropping frames.\n>\n> Fix this by holding image buffer in the queue during these conditions\n> until the\n> next embedded buffer dequeue event.\n>\n> Reported-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>\n> Signed-off-by: Naushir Patuck <naush@raspberrypi.com>\n\nReviewed-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>\nTested-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>\n\n> ---\n>  .../pipeline/raspberrypi/raspberrypi.cpp      | 19 ++++++++++++++-----\n>  1 file changed, 14 insertions(+), 5 deletions(-)\n>\n> diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> index 66a84b1dfb97..ef3c2d11d253 100644\n> --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> @@ -2159,16 +2159,12 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame\n> &bayerFrame, FrameBuffer *&em\n>         if (bayerQueue_.empty())\n>                 return false;\n>\n> -       /* Start with the front of the bayer queue. */\n> -       bayerFrame = std::move(bayerQueue_.front());\n> -       bayerQueue_.pop();\n> -\n>         /*\n>          * Find the embedded data buffer with a matching timestamp to pass\n> to\n>          * the IPA. Any embedded buffers with a timestamp lower than the\n>          * current bayer buffer will be removed and re-queued to the\n> driver.\n>          */\n> -       uint64_t ts = bayerFrame.buffer->metadata().timestamp;\n> +       uint64_t ts = bayerQueue_.front().buffer->metadata().timestamp;\n>         embeddedBuffer = nullptr;\n>         while (!embeddedQueue_.empty()) {\n>                 FrameBuffer *b = embeddedQueue_.front();\n> @@ -2188,10 +2184,23 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame\n> &bayerFrame, FrameBuffer *&em\n>         }\n>\n>         if (!embeddedBuffer && sensorMetadata_) {\n> +               if (embeddedQueue_.empty()) {\n> +                       /*\n> +                        * If the embedded buffer queue is empty, wait for\n> the next\n> +                        * buffer to arrive - dequeue ordering may send\n> the image\n> +                        * buffer first.\n> +                        */\n> +                       LOG(RPI, Debug) << \"Waiting for next embedded\n> buffer.\";\n> +                       return false;\n> +               }\n> +\n>                 /* Log if there is no matching embedded data buffer found.\n> */\n>                 LOG(RPI, Debug) << \"Returning bayer frame without a\n> matching embedded buffer.\";\n>         }\n>\n> +       bayerFrame = std::move(bayerQueue_.front());\n> +       bayerQueue_.pop();\n> +\n>         return true;\n>  }\n>\n> --\n> 2.25.1\n>\n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 55D9BBE173\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 13 Jul 2022 12:13:43 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id CD4F963312;\n\tWed, 13 Jul 2022 14:13:42 +0200 (CEST)","from mail-qk1-x736.google.com (mail-qk1-x736.google.com\n\t[IPv6:2607:f8b0:4864:20::736])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 0E45B6274E\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 13 Jul 2022 14:13:41 +0200 (CEST)","by mail-qk1-x736.google.com with SMTP id v6so8417995qkh.2\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 13 Jul 2022 05:13:40 -0700 (PDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1657714422;\n\tbh=64mecQ0jp72vuYlkj/4Z7yREWclKvL6ANDbOkTjqdsA=;\n\th=References:In-Reply-To:Date:To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=Mu0Fy8CNBKEInu7ifpYA7A7rQHD9CMMWZnms2L2HEEZzgBPkH+n92Ev8tvTjx7gpd\n\t+C2EORmPe12YM7VbgdDx6s04ulfDt3KDTpDvPM+OBbPQazMbuplf2245J+Ml1FMOZs\n\tJhotI0jDgEVDGGjLNK1M9VXwQql/Y7qdNJfQtYcQOVR/ywa2iy8NuThiH8nBe6OtON\n\tHFZ4FOht+i3xqAimupbbi3hiHqMPRqtELGCQ2I0F/5zPG1u17ibkbZXbFKeYP8gl41\n\tMnCyNwXuNfgTsM/CXONMnfloMYHiRbEyMnP46d/tc45L9li8ENZEOwKqz1iZ2WFWGj\n\tjEKEW2PvHT/WA==","v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=mime-version:references:in-reply-to:from:date:message-id:subject:to;\n\tbh=0BjnE6HbijetljTjb866hzO4we6xNYjOYWj+DnoffhU=;\n\tb=nojBOHKKQeb8+62aat+2Jv4YTHVrPGF6jQ6yD6ny/Y4lpwuYcuiLgMWqhTGJk4uOJc\n\tah6/SlaNH3Oj4qDpTdIzFnkd34PJTwV5ONHvmEAhfcOGyhUPRUFA7f1kKMFEOrBHpLfm\n\tLKl7YDku3N8ASk76GmGpkIsYf+Jr3w4Hm5laU+WOVvlMVD/wqDzbSO/yuwHBAQBj6E81\n\tNANeNKeGb4DOK7rV7cjQr9p39z4eU1UAdXOItPL9gdLK3AxwyXWXmv4tTCJ8hNaMkjkw\n\t2HrJSZT5KIK+DT1wY99sMmX0VJmVl/nM4rZgSerWiE3eefzKVoNTPxuI7wY2N9gs3tUS\n\tThRA=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=raspberrypi.com\n\theader.i=@raspberrypi.com\n\theader.b=\"nojBOHKK\"; dkim-atps=neutral","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=x-gm-message-state:mime-version:references:in-reply-to:from:date\n\t:message-id:subject:to;\n\tbh=0BjnE6HbijetljTjb866hzO4we6xNYjOYWj+DnoffhU=;\n\tb=2v7FDh65wFlL2NBfkBLtBjGwT9yxfli7EYw93+O4r/EuXi6/rMTc9XytrYxjLng25N\n\thi+vGCV5P1EPnlVm1llGJfZGsxg0vuTvIWlz1g+PUPDmfpIwGbmSnDFhhFos0wdgZ26y\n\tZyQ+4GjHx+VIRFegwZWOqA2wwAEl8vcI0fLNaI0whQMeSx9UljMAHfZDvWnoYAsG5UQb\n\tN8+7jNgN+bN44r2G9/jTa7pnMCtnZCPI0MnTdDNh/en9+QCzpDq6M19jJMgpIek4l6Tw\n\tkCqFhc9LxsCDqY91yavtbRLgj9aR8rdJ4iC6OK0SbP/p9PDfP5fP0FquzuC8Zn7ktCoM\n\tEQLA==","X-Gm-Message-State":"AJIora+Kwqydhbal9sVu2UaaD1vytJKKKYoZOl8nIhELqXXQF2RInpvf\n\t9C6m9JLYkKhg0OywGMJ5e5McdcM9e5fgDYeMpd+tdw==","X-Google-Smtp-Source":"AGRyM1ufO/ng9oGvFfBSunNUR1AtAMEX641yObko4nwNmXie7euk8zDfyPLFqLsI7mQFr3D220398Y8VfuJ7HqmH5B4=","X-Received":"by 2002:a05:620a:408e:b0:6a7:1815:a431 with SMTP id\n\tf14-20020a05620a408e00b006a71815a431mr1953198qko.551.1657714419818;\n\tWed, 13 Jul 2022 05:13:39 -0700 (PDT)","MIME-Version":"1.0","References":"<20220713120035.15604-1-naush@raspberrypi.com>","In-Reply-To":"<20220713120035.15604-1-naush@raspberrypi.com>","Date":"Wed, 13 Jul 2022 13:13:29 +0100","Message-ID":"<CAPhyPA40aGa7fTrRWXRBuSecANVx1SzwvOUbaxLt6H_P=d8nbA@mail.gmail.com>","To":"Naushir Patuck <naush@raspberrypi.com>,\n\tlibcamera-devel@lists.libcamera.org","Content-Type":"multipart/alternative; boundary=\"0000000000002201ac05e3aeb787\"","Subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","From":"Nick Hollinghurst via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>","Reply-To":"Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}},{"id":24013,"web_url":"https://patchwork.libcamera.org/comment/24013/","msgid":"<CAEmqJPqd20UZqiZzbKyT=FAb7AwasKfKYd7O-LCixPL8Ux8vVA@mail.gmail.com>","date":"2022-07-21T07:05:25","subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"Hi,\n\nAny takers for another review on this change please?\n\nRegards,\nNaush\n\nOn Wed, 13 Jul 2022 at 13:00, Naushir Patuck <naush@raspberrypi.com> wrote:\n\n> The logic used to match asynchronous image and embedded buffers was being\n> overly\n> aggressive by possibly allowing an unmatched image buffer to be sent to\n> the IPA\n> if the matching embedded buffer had not yet been dequeued. This condition\n> only\n> occurs when the system is heavily loaded and dropping frames.\n>\n> Fix this by holding image buffer in the queue during these conditions\n> until the\n> next embedded buffer dequeue event.\n>\n> Reported-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>\n> Signed-off-by: Naushir Patuck <naush@raspberrypi.com>\n> ---\n>  .../pipeline/raspberrypi/raspberrypi.cpp      | 19 ++++++++++++++-----\n>  1 file changed, 14 insertions(+), 5 deletions(-)\n>\n> diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> index 66a84b1dfb97..ef3c2d11d253 100644\n> --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> @@ -2159,16 +2159,12 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame\n> &bayerFrame, FrameBuffer *&em\n>         if (bayerQueue_.empty())\n>                 return false;\n>\n> -       /* Start with the front of the bayer queue. */\n> -       bayerFrame = std::move(bayerQueue_.front());\n> -       bayerQueue_.pop();\n> -\n>         /*\n>          * Find the embedded data buffer with a matching timestamp to pass\n> to\n>          * the IPA. Any embedded buffers with a timestamp lower than the\n>          * current bayer buffer will be removed and re-queued to the\n> driver.\n>          */\n> -       uint64_t ts = bayerFrame.buffer->metadata().timestamp;\n> +       uint64_t ts = bayerQueue_.front().buffer->metadata().timestamp;\n>         embeddedBuffer = nullptr;\n>         while (!embeddedQueue_.empty()) {\n>                 FrameBuffer *b = embeddedQueue_.front();\n> @@ -2188,10 +2184,23 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame\n> &bayerFrame, FrameBuffer *&em\n>         }\n>\n>         if (!embeddedBuffer && sensorMetadata_) {\n> +               if (embeddedQueue_.empty()) {\n> +                       /*\n> +                        * If the embedded buffer queue is empty, wait for\n> the next\n> +                        * buffer to arrive - dequeue ordering may send\n> the image\n> +                        * buffer first.\n> +                        */\n> +                       LOG(RPI, Debug) << \"Waiting for next embedded\n> buffer.\";\n> +                       return false;\n> +               }\n> +\n>                 /* Log if there is no matching embedded data buffer found.\n> */\n>                 LOG(RPI, Debug) << \"Returning bayer frame without a\n> matching embedded buffer.\";\n>         }\n>\n> +       bayerFrame = std::move(bayerQueue_.front());\n> +       bayerQueue_.pop();\n> +\n>         return true;\n>  }\n>\n> --\n> 2.25.1\n>\n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id BF723BD1F1\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 21 Jul 2022 07:05:44 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6871A63312;\n\tThu, 21 Jul 2022 09:05:44 +0200 (CEST)","from mail-lj1-x231.google.com (mail-lj1-x231.google.com\n\t[IPv6:2a00:1450:4864:20::231])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 17A56603F5\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 21 Jul 2022 09:05:42 +0200 (CEST)","by mail-lj1-x231.google.com with SMTP id w2so743264ljj.7\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 21 Jul 2022 00:05:42 -0700 (PDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1658387144;\n\tbh=8MC8U9YaQqXop3HTJBwuDHgsRqELXqNUaUEwotqRL7U=;\n\th=References:In-Reply-To:Date:To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=1P0VWBCOfVw76p6bZwuA3rkzLmp1eMpUKjmCcJ3IrHUs9mN/Jad28UeQIVhQBHxsQ\n\tBmEoDQTWIqyjkKVp9q0waLVxmIL3L+nhi8Qi3uvOMq6xt+npOpimWDVFGWZImEExZu\n\tALqGXr73vyOGY2VkBhi6nt003ufaqDGfIMvJatiKfy998oCPeUUbVMR1h9nLLAboYj\n\tesDvrcVnh5jYmBPf9gAJW2W3HO41G6lywyd6BnMLonueHTaU4iuvIZG+Gb3tW34/Vl\n\tLcYDp4k3pNOqwRMJJQcHOt+l6v8Ut8FJIrpuY/JuuDB6cGQZRTNSf/Nhu6wCX6jicY\n\t0Dm2K71UgGUGw==","v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google;\n\th=mime-version:references:in-reply-to:from:date:message-id:subject:to\n\t:cc; bh=Fcygieb3UehuvodIcQjSW4oiPi+49hWmfxRe25f9NGI=;\n\tb=F0k4RKvxCJr/tHgiMwChKi0QyFV4ZPRamFEdm8GCnRyqY4L6F+q5AMwaRx9wx0PD1D\n\tzbYs+Td7d2PgUnEpaOnIk046ir1mbRxrFnwIzQe9P/FQwyUUmeYc5VgZNpBNHqnNeUeV\n\teL3hsLd9ReFCuQWpt2xqpZfp7dO475rhkqCsHMrOn6xQeTD+NZz6m0EGj4eXcJmdDGwi\n\thb+mqb781Sy4WjR0Xg2cNHBQGdMH8E8Yja2pbubCG+UaXS+MUHTWGAiPv7yBnR1neMZ6\n\tH8qv/uLCiOa5dYNP84QCK+aJ1zE5/AcrE/A4AGNeNEjukVJLyX5ow1KIhA6LmrpDrJty\n\tmS1Q=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=raspberrypi.com\n\theader.i=@raspberrypi.com\n\theader.b=\"F0k4RKvx\"; dkim-atps=neutral","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=x-gm-message-state:mime-version:references:in-reply-to:from:date\n\t:message-id:subject:to:cc;\n\tbh=Fcygieb3UehuvodIcQjSW4oiPi+49hWmfxRe25f9NGI=;\n\tb=kwzc312OLMz0Op9rDcmy1XGW9dE3qhMmU/399I89Q9v9FCAjgwRkP7N1/Qd5qCEppA\n\tfafF/UtE0H5omRBXx2Mor6IseUeZsfPSX0mRTKDSADCaLd9GtuLCv5INivuv3afQbc+t\n\tDQcmXw3xgO4DER7MUagbaZzA7B0gr4kZbJ/nXnGOzGnlwKxWgisoNcoCx752XJsHqiE+\n\tMMqccRXsNGppHgkA9hX456+AB+8Mbu+pb0IR3cK/oW61LsDDNJSc08EcK2Vh6MO8LYfW\n\tdSsUIEW+NW3ZYluFxIokKureKKqCecPqxPB+pbePoyZ5/7ZmvjEGktco3cMqHhWTYsr/\n\tBXgQ==","X-Gm-Message-State":"AJIora8zlmSmk16Ys2w1rTkxPhrdO9pCEIhWBtSD8ogIaRo5d9Cp4ndK\n\tJ/gcrIcxrNnHXtCnPNqMAm77nLPNEW2ewMjOHcR+m4ISEhZrdw==","X-Google-Smtp-Source":"AGRyM1sZmEfE1UB5idm2AYI7uFV+KGT2GH36ZkH87yph5byltX3dFJPAhz2DWh9z0RwkFgynOngiGzpsG7fg1l0kcoM=","X-Received":"by 2002:a2e:8785:0:b0:25d:473b:b60f with SMTP id\n\tn5-20020a2e8785000000b0025d473bb60fmr18675215lji.354.1658387140963;\n\tThu, 21 Jul 2022 00:05:40 -0700 (PDT)","MIME-Version":"1.0","References":"<20220713120035.15604-1-naush@raspberrypi.com>","In-Reply-To":"<20220713120035.15604-1-naush@raspberrypi.com>","Date":"Thu, 21 Jul 2022 08:05:25 +0100","Message-ID":"<CAEmqJPqd20UZqiZzbKyT=FAb7AwasKfKYd7O-LCixPL8Ux8vVA@mail.gmail.com>","To":"libcamera devel <libcamera-devel@lists.libcamera.org>","Content-Type":"multipart/alternative; boundary=\"00000000000070059105e44b5895\"","Subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","From":"Naushir Patuck via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>","Reply-To":"Naushir Patuck <naush@raspberrypi.com>","Cc":"Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}},{"id":24042,"web_url":"https://patchwork.libcamera.org/comment/24042/","msgid":"<YtnH3wg59uBzqzQu@pendragon.ideasonboard.com>","date":"2022-07-21T21:40:47","subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Naush,\n\nThank you for the patch.\n\nOn Wed, Jul 13, 2022 at 01:00:35PM +0100, Naushir Patuck via libcamera-devel wrote:\n> The logic used to match asynchronous image and embedded buffers was being overly\n> aggressive by possibly allowing an unmatched image buffer to be sent to the IPA\n> if the matching embedded buffer had not yet been dequeued. This condition only\n> occurs when the system is heavily loaded and dropping frames.\n> \n> Fix this by holding image buffer in the queue during these conditions until the\n> next embedded buffer dequeue event.\n> \n> Reported-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>\n> Signed-off-by: Naushir Patuck <naush@raspberrypi.com>\n> ---\n>  .../pipeline/raspberrypi/raspberrypi.cpp      | 19 ++++++++++++++-----\n>  1 file changed, 14 insertions(+), 5 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> index 66a84b1dfb97..ef3c2d11d253 100644\n> --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> @@ -2159,16 +2159,12 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame &bayerFrame, FrameBuffer *&em\n>  \tif (bayerQueue_.empty())\n>  \t\treturn false;\n\nNot strictly related to this patch, but this can never happen, as it's\ntested by the caller.\n\n>  \n> -\t/* Start with the front of the bayer queue. */\n> -\tbayerFrame = std::move(bayerQueue_.front());\n> -\tbayerQueue_.pop();\n> -\n>  \t/*\n>  \t * Find the embedded data buffer with a matching timestamp to pass to\n>  \t * the IPA. Any embedded buffers with a timestamp lower than the\n>  \t * current bayer buffer will be removed and re-queued to the driver.\n>  \t */\n> -\tuint64_t ts = bayerFrame.buffer->metadata().timestamp;\n> +\tuint64_t ts = bayerQueue_.front().buffer->metadata().timestamp;\n>  \tembeddedBuffer = nullptr;\n>  \twhile (!embeddedQueue_.empty()) {\n>  \t\tFrameBuffer *b = embeddedQueue_.front();\n> @@ -2188,10 +2184,23 @@ bool RPiCameraData::findMatchingBuffers(BayerFrame &bayerFrame, FrameBuffer *&em\n>  \t}\n>  \n>  \tif (!embeddedBuffer && sensorMetadata_) {\n> +\t\tif (embeddedQueue_.empty()) {\n> +\t\t\t/*\n> +\t\t\t * If the embedded buffer queue is empty, wait for the next\n> +\t\t\t * buffer to arrive - dequeue ordering may send the image\n> +\t\t\t * buffer first.\n> +\t\t\t */\n> +\t\t\tLOG(RPI, Debug) << \"Waiting for next embedded buffer.\";\n> +\t\t\treturn false;\n> +\t\t}\n> +\n>  \t\t/* Log if there is no matching embedded data buffer found. */\n>  \t\tLOG(RPI, Debug) << \"Returning bayer frame without a matching embedded buffer.\";\n>  \t}\n>  \n> +\tbayerFrame = std::move(bayerQueue_.front());\n> +\tbayerQueue_.pop();\n> +\n\nThis looks fine, but I wonder if it could be simplified by first\nhandling the case where the sensor doesn't produce metadata (possibly in\nRPiCameraData::tryRunPipeline()). This can be done on top, so I'll apply\nthis patch with\n\nReviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n\n>  \treturn true;\n>  }\n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 8389EBE173\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 21 Jul 2022 21:40:52 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id E2C626330F;\n\tThu, 21 Jul 2022 23:40:51 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 48FC9601B8\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 21 Jul 2022 23:40:50 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C0DC7496;\n\tThu, 21 Jul 2022 23:40:49 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1658439651;\n\tbh=zRIqTscPhVzPmxXIMifAWrZ4f6yBj9xmRokcv+6bxtk=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=UnumL9btaTUfGpO4NcmxVcbqMPG8vMDM/WAAd7dnYOtsj4T9e88vuq7eCFBV1Mub+\n\tQuP2id6dN/8LUUxvhjuAXW6/wujYkuEup+erzVhuyrL7PRmfFBdWt8WqppRuvnECet\n\t/k/YZ7TTH46k3fcee50jzMEJOtEciA2tfSljS1Q6FKrkaB+KT0QeOP8wTNeOHEFCgP\n\tMQk9UHOMVpmPkpDSzbNjKnvGsLeQ0SzJtN+RJy7w4jk4EyubpI6bEBnuqVyWLxMHli\n\t0kF7nG/Js2dvYXKz9HLFlyKVHCWAftuYbpJQlFTKu0F+SJy+Tyyy6bGAA3hMVqX04f\n\t8wQqt6vaVw/LQ==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1658439649;\n\tbh=zRIqTscPhVzPmxXIMifAWrZ4f6yBj9xmRokcv+6bxtk=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=oIGcEIgAGg16/jNc7gg7Tw90vOAjKGALlt8Hq6oCmSpTXNa3kk81q8cxtW/3NGyno\n\tXjBjCXA+jwTHA6nAt0zTa0mKT6fTMwrJSyj0lv5nkEyfvp/7B9kUi3mTil9pZrJhYu\n\tijsToZnx35ezGjS9887v5c9Mt/XrtVEUGpHh40BE="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"oIGcEIgA\"; dkim-atps=neutral","Date":"Fri, 22 Jul 2022 00:40:47 +0300","To":"Naushir Patuck <naush@raspberrypi.com>","Message-ID":"<YtnH3wg59uBzqzQu@pendragon.ideasonboard.com>","References":"<20220713120035.15604-1-naush@raspberrypi.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20220713120035.15604-1-naush@raspberrypi.com>","Subject":"Re: [libcamera-devel] [PATCH] pipeline: raspberrypi: Improve\n\timage/embedded buffer matching logic","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","From":"Laurent Pinchart via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>","Reply-To":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","Cc":"Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>,\n\tlibcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]