[{"id":10803,"web_url":"https://patchwork.libcamera.org/comment/10803/","msgid":"<20200624173804.GC1595450@oden.dyn.berto.se>","date":"2020-06-24T17:38:04","subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","submitter":{"id":5,"url":"https://patchwork.libcamera.org/api/people/5/","name":"Niklas Söderlund","email":"niklas.soderlund@ragnatech.se"},"content":"Hi Paul,\n\nThanks for your work.\n\nOn 2020-06-24 23:07:52 +0900, Paul Elder wrote:\n> Test the V4L2 compatibility layer by running v4l2-compliance -s on every\n> /dev/video* device.\n\nI wonder if this is not a tad to aggressive. Imagine the test being run \non a device where one video node is not covered by a libcamera pipeline \nand also does not pass v4l2-compliance. Would this not lead to the \nlibcamera test suite to fail?\n\nWould it be possible you think to check the driver name of the video \ndevice and check it against an whitelist (vimc, vivid, ?) before running \nthe v4l2-compliance suite?\n\nYou can get the driver name from\n\n    v4l2-ctl -D -d /dev/videoX\n\n> \n> Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>\n> ---\n>  test/meson.build                     |   1 +\n>  test/v4l2_compat/meson.build         |  10 +++\n>  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++\n>  3 files changed, 136 insertions(+)\n>  create mode 100644 test/v4l2_compat/meson.build\n>  create mode 100755 test/v4l2_compat/v4l2_compat_test.py\n> \n> diff --git a/test/meson.build b/test/meson.build\n> index a868813..591920f 100644\n> --- a/test/meson.build\n> +++ b/test/meson.build\n> @@ -12,6 +12,7 @@ subdir('pipeline')\n>  subdir('process')\n>  subdir('serialization')\n>  subdir('stream')\n> +subdir('v4l2_compat')\n>  subdir('v4l2_subdevice')\n>  subdir('v4l2_videodevice')\n>  \n> diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> new file mode 100644\n> index 0000000..a67aac4\n> --- /dev/null\n> +++ b/test/v4l2_compat/meson.build\n> @@ -0,0 +1,10 @@\n> +# SPDX-License-Identifier: CC0-1.0\n> +\n> +pymod = import('python')\n> +py3 = pymod.find_installation('python3')\n> +\n> +v4l2_compat_test = files('v4l2_compat_test.py')\n> +\n> +test('v4l2_compat_test', py3,\n> +     args : v4l2_compat_test,\n> +     suite : 'v4l2_compat')\n> diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> new file mode 100755\n> index 0000000..f56db4e\n> --- /dev/null\n> +++ b/test/v4l2_compat/v4l2_compat_test.py\n> @@ -0,0 +1,125 @@\n> +#!/usr/bin/env python3\n> +# SPDX-License-Identifier: GPL-2.0-or-later\n> +# Copyright (C) 2019, Google Inc.\n> +#\n> +# Author: Paul Elder <paul.elder@ideasonboard.com>\n> +#\n> +# v4l2_compat_test.py - Test the V4L2 compatibility layer\n> +\n> +import os\n> +import re\n> +from shutil import which\n> +import subprocess\n> +import sys\n> +\n> +TestPass = 0\n> +TestFail = -1\n> +TestSkip = 77\n> +\n> +\n> +supported_pipelines = [\n> +    \"uvcvideo\",\n> +    \"vimc\",\n> +]\n> +\n> +\n> +def find_file(name, path):\n> +    for root, dirs, files in os.walk(path):\n> +        if name in files:\n> +            return os.path.join(root, name)\n> +\n> +\n> +def grep(exp, arr):\n> +    return [s for s in arr if re.search(exp, s)]\n> +\n> +\n> +def run_with_stdout(cmd, args=\"\", env={}):\n> +    try:\n> +        with open(os.devnull, 'w') as devnull:\n> +            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> +    except subprocess.CalledProcessError as err:\n> +        output = err.output\n> +    return output.decode(\"utf-8\").split(\"\\n\")\n> +\n> +\n> +def extract_result(result):\n> +    res = result.split(\", \")\n> +    ret = {}\n> +    ret[\"total\"]     = int(res[0].split(\": \")[-1])\n> +    ret[\"succeeded\"] = int(res[1].split(\": \")[-1])\n> +    ret[\"failed\"]    = int(res[2].split(\": \")[-1])\n> +    ret[\"warnings\"]  = int(res[3].split(\": \")[-1])\n> +    ret[\"device\"]    = res[0].split()[4].strip(\":\")\n> +    ret[\"driver\"]    = res[0].split()[2]\n> +    return ret\n> +\n> +\n> +def print_output_arr(output_arr):\n> +    print(\"\\n\".join(output_arr))\n> +\n> +\n> +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> +    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> +    result = extract_result(output[-2])\n> +    if result[\"driver\"] != \"libcamera\":\n> +        return TestSkip\n> +\n> +    if result['failed'] == 0:\n> +        return TestPass\n> +\n> +    # vimc will fail s_fmt because it only supports framesizes that are\n> +    # multiples of 3\n> +    if base_driver == \"vimc\" and result[\"failed\"] <= 1:\n> +        failures = grep(\"fail\", output)\n> +        if re.search(\"S_FMT cannot handle an invalid format\", failures[0]) is None:\n> +            print_output_arr(output)\n> +            return TestFail\n> +        return TestPass\n> +\n> +    print_output_arr(output)\n> +    return TestFail\n> +\n> +\n> +def main():\n> +    v4l2_compliance = which(\"v4l2-compliance\")\n> +    if v4l2_compliance is None:\n> +        print(\"v4l2-compliance is not available\")\n> +        return TestSkip\n> +\n> +    v4l2_ctl = which(\"v4l2-ctl\")\n> +    if v4l2_ctl is None:\n> +        print(\"v4l2-ctl is not available\")\n> +        return TestSkip\n> +\n> +    v4l2_compat = find_file(\"v4l2-compat.so\", \".\")\n> +    if v4l2_compat is None:\n> +        print(\"v4l2-compat.so is not built\")\n> +        return TestSkip\n> +\n> +    dev_nodes = grep(\"video\", os.listdir(\"/dev\"))\n> +    dev_nodes = list(map(lambda s: f\"/dev/{s}\", dev_nodes))\n> +    if len(dev_nodes) == 0:\n> +        print(\"no video nodes available to test with\")\n> +        return TestSkip\n> +\n> +    failed = []\n> +    for device in dev_nodes:\n> +        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> +        driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n> +        if driver not in supported_pipelines:\n> +            continue\n> +\n> +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)\n> +        if ret == TestFail:\n> +            failed.append(device)\n> +\n> +    if len(failed) > 0:\n> +        print(f\"Failed {len(failed)} tests:\")\n> +        for device in failed:\n> +            print(f\"- {device}\")\n> +\n> +    return TestPass if not failed else TestFail\n> +\n> +\n> +if __name__ == '__main__':\n> +    sys.exit(main())\n> -- \n> 2.27.0\n> \n> _______________________________________________\n> libcamera-devel mailing list\n> libcamera-devel@lists.libcamera.org\n> https://lists.libcamera.org/listinfo/libcamera-devel","headers":{"Return-Path":"<niklas.soderlund@ragnatech.se>","Received":["from mail-lf1-x142.google.com (mail-lf1-x142.google.com\n\t[IPv6:2a00:1450:4864:20::142])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 32220603BA\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 24 Jun 2020 19:38:07 +0200 (CEST)","by mail-lf1-x142.google.com with SMTP id c11so1723274lfh.8\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 24 Jun 2020 10:38:07 -0700 (PDT)","from localhost (h-209-203.A463.priv.bahnhof.se. [155.4.209.203])\n\tby smtp.gmail.com with ESMTPSA id\n\tw26sm4280770ljj.114.2020.06.24.10.38.05\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tWed, 24 Jun 2020 10:38:05 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected)\n\theader.d=ragnatech-se.20150623.gappssmtp.com\n\theader.i=@ragnatech-se.20150623.gappssmtp.com header.b=\"BaKRISca\"; \n\tdkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=ragnatech-se.20150623.gappssmtp.com; s=20150623;\n\th=date:from:to:cc:subject:message-id:references:mime-version\n\t:content-disposition:content-transfer-encoding:in-reply-to;\n\tbh=dWlm4QBDvINaKJpf+k96lr3Z44FMgWVaalvDArHyHjk=;\n\tb=BaKRIScaAnWn8XArIzlO260xbklr/afpqB8ANdH21vWLtYQ9kMqnjAlpLx34APxZYC\n\tKXVqP6Smzzim5R7oMjpzvRFXSuiKgKPSkjfdUUFDt1KOiPXReyjAdPEfLQ75lZKcNval\n\tqX6orSBhKMz3EV7m0ZwpfbqOBZJmsB6KmyqNl1nOdA83wqaJH3gljUFQLmj5+DcxsTjK\n\tH+frJ0UWGmTywdz9w9FCRkW9+zOYHUq7cGyNRIhhRCtec01ybTUTN+oS1yhcXBxgbxe2\n\t2l/H8a8Jk16gvbf+RAKPmWTCZ5WLK8ACzt5vkyxEtExxEvkdxkEWuPTzDCKIpp3BfaJF\n\t2DFg==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:date:from:to:cc:subject:message-id:references\n\t:mime-version:content-disposition:content-transfer-encoding\n\t:in-reply-to;\n\tbh=dWlm4QBDvINaKJpf+k96lr3Z44FMgWVaalvDArHyHjk=;\n\tb=HwK3utJXqC98zQptTtlQ3QE6rUCdoog1LjOcUQH0VaIh8OmW17GUUafPqJOSMYi1Mg\n\tJbUjxZUrknd4SYtOuJcStkUwsciHXEAMQOPQJ0R2gWV8hPpzvjEMZCYn+LXqH8dFYGr7\n\tBVWpdiANWRbyRtZKGKza1IT2GVMXAlVO3PdYaN1j86TldQVwK+ModRL/NJUdnN4mwP0d\n\tlipTaBaZIYWxtY5S5O3Yw28XyaWFEtL6X5hN4qpVA4GNn8VyLIqNU6kZWlA5Sweks2aR\n\tcLKRJSdRcvGz1exkS5MsX7Id5fO6BamiMM3bDwSXBRyXRefYQJy64OeOObWT1Hi86ZAk\n\tSICw==","X-Gm-Message-State":"AOAM533bI5/rnXpALGEqDvxzIN6XSydgEBqHBrPzU4ghD3h5FLIvwwRp\n\tPySQ0xwkBzFaNu/jBdJlmaNOmcZmVJc=","X-Google-Smtp-Source":"ABdhPJwenS15FtF/gs6joOUJQmeEQm81cy4qLkQ/pFy1H+4uRvyXjrzjBpqmLFzw+nrMc8ZSy3BN2Q==","X-Received":"by 2002:a19:7d84:: with SMTP id\n\ty126mr16122470lfc.149.1593020285887; \n\tWed, 24 Jun 2020 10:38:05 -0700 (PDT)","Date":"Wed, 24 Jun 2020 19:38:04 +0200","From":"Niklas =?iso-8859-1?q?S=F6derlund?= <niklas.soderlund@ragnatech.se>","To":"Paul Elder <paul.elder@ideasonboard.com>","Cc":"libcamera-devel@lists.libcamera.org","Message-ID":"<20200624173804.GC1595450@oden.dyn.berto.se>","References":"<20200624140752.46066-1-paul.elder@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=iso-8859-1","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200624140752.46066-1-paul.elder@ideasonboard.com>","Subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","X-List-Received-Date":"Wed, 24 Jun 2020 17:38:07 -0000"}},{"id":10807,"web_url":"https://patchwork.libcamera.org/comment/10807/","msgid":"<20200624222641.GI5980@pendragon.ideasonboard.com>","date":"2020-06-24T22:26:41","subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Niklas and Paul,\n\nOn Wed, Jun 24, 2020 at 07:38:04PM +0200, Niklas Söderlund wrote:\n> Hi Paul,\n> \n> Thanks for your work.\n> \n> On 2020-06-24 23:07:52 +0900, Paul Elder wrote:\n> > Test the V4L2 compatibility layer by running v4l2-compliance -s on every\n> > /dev/video* device.\n> \n> I wonder if this is not a tad to aggressive. Imagine the test being run \n> on a device where one video node is not covered by a libcamera pipeline \n> and also does not pass v4l2-compliance. Would this not lead to the \n> libcamera test suite to fail?\n> \n> Would it be possible you think to check the driver name of the video \n> device and check it against an whitelist (vimc, vivid, ?) before running \n> the v4l2-compliance suite?\n> \n> You can get the driver name from\n> \n>     v4l2-ctl -D -d /dev/videoX\n\nIsn't this exactly what the code below is doing ? :-)\n\n> > Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>\n> > ---\n> >  test/meson.build                     |   1 +\n> >  test/v4l2_compat/meson.build         |  10 +++\n> >  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++\n> >  3 files changed, 136 insertions(+)\n> >  create mode 100644 test/v4l2_compat/meson.build\n> >  create mode 100755 test/v4l2_compat/v4l2_compat_test.py\n> > \n> > diff --git a/test/meson.build b/test/meson.build\n> > index a868813..591920f 100644\n> > --- a/test/meson.build\n> > +++ b/test/meson.build\n> > @@ -12,6 +12,7 @@ subdir('pipeline')\n> >  subdir('process')\n> >  subdir('serialization')\n> >  subdir('stream')\n> > +subdir('v4l2_compat')\n> >  subdir('v4l2_subdevice')\n> >  subdir('v4l2_videodevice')\n> >  \n> > diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> > new file mode 100644\n> > index 0000000..a67aac4\n> > --- /dev/null\n> > +++ b/test/v4l2_compat/meson.build\n> > @@ -0,0 +1,10 @@\n> > +# SPDX-License-Identifier: CC0-1.0\n> > +\n> > +pymod = import('python')\n> > +py3 = pymod.find_installation('python3')\n> > +\n> > +v4l2_compat_test = files('v4l2_compat_test.py')\n> > +\n> > +test('v4l2_compat_test', py3,\n> > +     args : v4l2_compat_test,\n> > +     suite : 'v4l2_compat')\n\nWould this be simpler ?\n\ntest('v4l2_compat_test', v4l2_compat_test,\n     suite : 'v4l2_compat')\n\n> > diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> > new file mode 100755\n> > index 0000000..f56db4e\n> > --- /dev/null\n> > +++ b/test/v4l2_compat/v4l2_compat_test.py\n> > @@ -0,0 +1,125 @@\n> > +#!/usr/bin/env python3\n> > +# SPDX-License-Identifier: GPL-2.0-or-later\n> > +# Copyright (C) 2019, Google Inc.\n\n2020 ?\n\n> > +#\n> > +# Author: Paul Elder <paul.elder@ideasonboard.com>\n> > +#\n> > +# v4l2_compat_test.py - Test the V4L2 compatibility layer\n> > +\n> > +import os\n> > +import re\n> > +from shutil import which\n\nI'd just import shutil and use shutil.which, up to you.\n\n> > +import subprocess\n> > +import sys\n> > +\n> > +TestPass = 0\n> > +TestFail = -1\n> > +TestSkip = 77\n> > +\n> > +\n> > +supported_pipelines = [\n> > +    \"uvcvideo\",\n\nOur python scripts usually use single quotes for strings.\n\n> > +    \"vimc\",\n> > +]\n> > +\n> > +\n> > +def find_file(name, path):\n> > +    for root, dirs, files in os.walk(path):\n> > +        if name in files:\n> > +            return os.path.join(root, name)\n\nThat's a bit of heavy artillery. How about passing the path to\nv4l2-compat.so to the test ? Something along the lines of\n\ndiff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\nindex a67aac41b273..e24d10bd327c 100644\n--- a/test/v4l2_compat/meson.build\n+++ b/test/v4l2_compat/meson.build\n@@ -1,10 +1,9 @@\n # SPDX-License-Identifier: CC0-1.0\n\n-pymod = import('python')\n-py3 = pymod.find_installation('python3')\n+if is_variable('v4l2_compat')\n+    v4l2_compat_test = files('v4l2_compat_test.py')\n\n-v4l2_compat_test = files('v4l2_compat_test.py')\n-\n-test('v4l2_compat_test', py3,\n-     args : v4l2_compat_test,\n-     suite : 'v4l2_compat')\n+    test('v4l2_compat_test', v4l2_compat_test,\n+         args : v4l2_compat,\n+         suite : 'v4l2_compat')\n+endif\n\n> > +\n> > +\n> > +def grep(exp, arr):\n> > +    return [s for s in arr if re.search(exp, s)]\n> > +\n> > +\n> > +def run_with_stdout(cmd, args=\"\", env={}):\n> > +    try:\n> > +        with open(os.devnull, 'w') as devnull:\n> > +            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n\nDo you need to run this through a shell ? On my system, one of the video\nnodes related to my integrated UVC webcam isn't handled by libcamera.\nWhen v4l2-compliance is run on it with LD_PRELOAD=v4l2-compat.so, it\nhangs if run through a shell, and doesn't otherwise. I suspect it's the\nshell hanging, but if we don't need shell=True, then we may skip the\ninvestigation :-)\n\nYou will need to turn the first argument into an array. I've tried the\nfollowing, it seems to work.\n\ndiff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\nindex f56db4e91890..d05b3235a78c 100755\n--- a/test/v4l2_compat/v4l2_compat_test.py\n+++ b/test/v4l2_compat/v4l2_compat_test.py\n@@ -33,10 +33,10 @@ def grep(exp, arr):\n     return [s for s in arr if re.search(exp, s)]\n \n \n-def run_with_stdout(cmd, args=\"\", env={}):\n+def run_with_stdout(*args, env={}):\n     try:\n         with open(os.devnull, 'w') as devnull:\n-            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n+            output = subprocess.check_output(args, env=env, stderr=devnull)\n     except subprocess.CalledProcessError as err:\n         output = err.output\n     return output.decode(\"utf-8\").split(\"\\n\")\n@@ -59,7 +59,7 @@ def print_output_arr(output_arr):\n \n \n def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n-    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n+    output = run_with_stdout(v4l2_compliance, \"-s\", \"-d\", device, env={\"LD_PRELOAD\": v4l2_compat})\n     result = extract_result(output[-2])\n     if result[\"driver\"] != \"libcamera\":\n         return TestSkip\n@@ -104,7 +104,7 @@ def main():\n \n     failed = []\n     for device in dev_nodes:\n-        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n+        out = run_with_stdout(v4l2_ctl, \"-D\", \"-d\", device)\n         driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n         if driver not in supported_pipelines:\n             continue\n\nAnd after further testing, it seems the test stills hangs from time to\ntime :-S I'll let you investigate if you can reproduce the issue.\n\nI'm also getting a floating point exception when I run v4l2-compliance\nwith LD_PRELOAD on my UVC webcam. Not too surprising as v4l2-compat\nreports\n\nWARN V4L2Compat v4l2_camera_proxy.cpp:515 sizeimage of at least one format is zero. Streaming this format will cause a floating point exception.\n\nThe camera produces formats::R8, which isn't listed in pixelFormatInfo.\nI think that will be fixed when moving pixelFormatInfo out of the\nv4l2-compat layer, like we've discussed offline.\n\n> > +    except subprocess.CalledProcessError as err:\n> > +        output = err.output\n> > +    return output.decode(\"utf-8\").split(\"\\n\")\n> > +\n> > +\n> > +def extract_result(result):\n> > +    res = result.split(\", \")\n> > +    ret = {}\n> > +    ret[\"total\"]     = int(res[0].split(\": \")[-1])\n> > +    ret[\"succeeded\"] = int(res[1].split(\": \")[-1])\n> > +    ret[\"failed\"]    = int(res[2].split(\": \")[-1])\n> > +    ret[\"warnings\"]  = int(res[3].split(\": \")[-1])\n> > +    ret[\"device\"]    = res[0].split()[4].strip(\":\")\n> > +    ret[\"driver\"]    = res[0].split()[2]\n> > +    return ret\n> > +\n> > +\n> > +def print_output_arr(output_arr):\n> > +    print(\"\\n\".join(output_arr))\n> > +\n> > +\n> > +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> > +    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> > +    result = extract_result(output[-2])\n> > +    if result[\"driver\"] != \"libcamera\":\n> > +        return TestSkip\n> > +\n> > +    if result['failed'] == 0:\n> > +        return TestPass\n> > +\n> > +    # vimc will fail s_fmt because it only supports framesizes that are\n> > +    # multiples of 3\n> > +    if base_driver == \"vimc\" and result[\"failed\"] <= 1:\n> > +        failures = grep(\"fail\", output)\n> > +        if re.search(\"S_FMT cannot handle an invalid format\", failures[0]) is None:\n\nIs there a reason not to use your grep function ?\n\n> > +            print_output_arr(output)\n> > +            return TestFail\n> > +        return TestPass\n> > +\n> > +    print_output_arr(output)\n> > +    return TestFail\n> > +\n> > +\n> > +def main():\n> > +    v4l2_compliance = which(\"v4l2-compliance\")\n> > +    if v4l2_compliance is None:\n> > +        print(\"v4l2-compliance is not available\")\n> > +        return TestSkip\n> > +\n> > +    v4l2_ctl = which(\"v4l2-ctl\")\n> > +    if v4l2_ctl is None:\n> > +        print(\"v4l2-ctl is not available\")\n> > +        return TestSkip\n> > +\n> > +    v4l2_compat = find_file(\"v4l2-compat.so\", \".\")\n> > +    if v4l2_compat is None:\n> > +        print(\"v4l2-compat.so is not built\")\n> > +        return TestSkip\n> > +\n> > +    dev_nodes = grep(\"video\", os.listdir(\"/dev\"))\n\nimport glob\n\n    dev_nodes = glob.glob(\"/dev/video*\")\n\n> > +    dev_nodes = list(map(lambda s: f\"/dev/{s}\", dev_nodes))\n\nOr the more Pythonic syntax:\n\n    dev_nodes = [f\"/dev/{s]\" for s in dev_nodes]\n\n> > +    if len(dev_nodes) == 0:\n> > +        print(\"no video nodes available to test with\")\n> > +        return TestSkip\n> > +\n> > +    failed = []\n> > +    for device in dev_nodes:\n> > +        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> > +        driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n> > +        if driver not in supported_pipelines:\n> > +            continue\n> > +\n> > +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)\n> > +        if ret == TestFail:\n> > +            failed.append(device)\n> > +\n> > +    if len(failed) > 0:\n> > +        print(f\"Failed {len(failed)} tests:\")\n> > +        for device in failed:\n> > +            print(f\"- {device}\")\n> > +\n> > +    return TestPass if not failed else TestFail\n> > +\n> > +\n> > +if __name__ == '__main__':\n> > +    sys.exit(main())","headers":{"Return-Path":"<laurent.pinchart@ideasonboard.com>","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CB327603BB\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 25 Jun 2020 00:26:42 +0200 (CEST)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 33DB62A8;\n\tThu, 25 Jun 2020 00:26:42 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"hMyoT3L2\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1593037602;\n\tbh=7T0wQqy0Z4Cmn2yrMqVmGlR5dUEy60HjliGB6O6EP2M=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=hMyoT3L2idfbkCF7HMoNf/oS7dJdFoT7FH7wgul6L8co8nJEspUp0OYdOvKnwtNkj\n\tNTIvm3xmAGzqC0zpOk/IYnlB0+5qbxPXZQs0MbDLEXcLZANhNI1Bge7Oo0XrdXeroK\n\tKxUkq7z61AgxNcm6JIbPMPZYn4AM2eC0+EOskawc=","Date":"Thu, 25 Jun 2020 01:26:41 +0300","From":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","To":"Niklas =?utf-8?q?S=C3=B6derlund?= <niklas.soderlund@ragnatech.se>","Cc":"Paul Elder <paul.elder@ideasonboard.com>,\n\tlibcamera-devel@lists.libcamera.org","Message-ID":"<20200624222641.GI5980@pendragon.ideasonboard.com>","References":"<20200624140752.46066-1-paul.elder@ideasonboard.com>\n\t<20200624173804.GC1595450@oden.dyn.berto.se>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200624173804.GC1595450@oden.dyn.berto.se>","Subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","X-List-Received-Date":"Wed, 24 Jun 2020 22:26:43 -0000"}},{"id":10808,"web_url":"https://patchwork.libcamera.org/comment/10808/","msgid":"<20200624223630.GA10256@oden.dyn.berto.se>","date":"2020-06-24T22:36:30","subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","submitter":{"id":5,"url":"https://patchwork.libcamera.org/api/people/5/","name":"Niklas Söderlund","email":"niklas.soderlund@ragnatech.se"},"content":"On 2020-06-25 01:26:41 +0300, Laurent Pinchart wrote:\n> Hi Niklas and Paul,\n> \n> On Wed, Jun 24, 2020 at 07:38:04PM +0200, Niklas Söderlund wrote:\n> > Hi Paul,\n> > \n> > Thanks for your work.\n> > \n> > On 2020-06-24 23:07:52 +0900, Paul Elder wrote:\n> > > Test the V4L2 compatibility layer by running v4l2-compliance -s on every\n> > > /dev/video* device.\n> > \n> > I wonder if this is not a tad to aggressive. Imagine the test being run \n> > on a device where one video node is not covered by a libcamera pipeline \n> > and also does not pass v4l2-compliance. Would this not lead to the \n> > libcamera test suite to fail?\n> > \n> > Would it be possible you think to check the driver name of the video \n> > device and check it against an whitelist (vimc, vivid, ?) before running \n> > the v4l2-compliance suite?\n> > \n> > You can get the driver name from\n> > \n> >     v4l2-ctl -D -d /dev/videoX\n> \n> Isn't this exactly what the code below is doing ? :-)\n\nYou are correct, sorry for the noise.\n\nDo you expect me to read the code as well as the commit message? :-P\n\n> \n> > > Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>\n> > > ---\n> > >  test/meson.build                     |   1 +\n> > >  test/v4l2_compat/meson.build         |  10 +++\n> > >  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++\n> > >  3 files changed, 136 insertions(+)\n> > >  create mode 100644 test/v4l2_compat/meson.build\n> > >  create mode 100755 test/v4l2_compat/v4l2_compat_test.py\n> > > \n> > > diff --git a/test/meson.build b/test/meson.build\n> > > index a868813..591920f 100644\n> > > --- a/test/meson.build\n> > > +++ b/test/meson.build\n> > > @@ -12,6 +12,7 @@ subdir('pipeline')\n> > >  subdir('process')\n> > >  subdir('serialization')\n> > >  subdir('stream')\n> > > +subdir('v4l2_compat')\n> > >  subdir('v4l2_subdevice')\n> > >  subdir('v4l2_videodevice')\n> > >  \n> > > diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> > > new file mode 100644\n> > > index 0000000..a67aac4\n> > > --- /dev/null\n> > > +++ b/test/v4l2_compat/meson.build\n> > > @@ -0,0 +1,10 @@\n> > > +# SPDX-License-Identifier: CC0-1.0\n> > > +\n> > > +pymod = import('python')\n> > > +py3 = pymod.find_installation('python3')\n> > > +\n> > > +v4l2_compat_test = files('v4l2_compat_test.py')\n> > > +\n> > > +test('v4l2_compat_test', py3,\n> > > +     args : v4l2_compat_test,\n> > > +     suite : 'v4l2_compat')\n> \n> Would this be simpler ?\n> \n> test('v4l2_compat_test', v4l2_compat_test,\n>      suite : 'v4l2_compat')\n> \n> > > diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> > > new file mode 100755\n> > > index 0000000..f56db4e\n> > > --- /dev/null\n> > > +++ b/test/v4l2_compat/v4l2_compat_test.py\n> > > @@ -0,0 +1,125 @@\n> > > +#!/usr/bin/env python3\n> > > +# SPDX-License-Identifier: GPL-2.0-or-later\n> > > +# Copyright (C) 2019, Google Inc.\n> \n> 2020 ?\n> \n> > > +#\n> > > +# Author: Paul Elder <paul.elder@ideasonboard.com>\n> > > +#\n> > > +# v4l2_compat_test.py - Test the V4L2 compatibility layer\n> > > +\n> > > +import os\n> > > +import re\n> > > +from shutil import which\n> \n> I'd just import shutil and use shutil.which, up to you.\n> \n> > > +import subprocess\n> > > +import sys\n> > > +\n> > > +TestPass = 0\n> > > +TestFail = -1\n> > > +TestSkip = 77\n> > > +\n> > > +\n> > > +supported_pipelines = [\n> > > +    \"uvcvideo\",\n> \n> Our python scripts usually use single quotes for strings.\n> \n> > > +    \"vimc\",\n> > > +]\n> > > +\n> > > +\n> > > +def find_file(name, path):\n> > > +    for root, dirs, files in os.walk(path):\n> > > +        if name in files:\n> > > +            return os.path.join(root, name)\n> \n> That's a bit of heavy artillery. How about passing the path to\n> v4l2-compat.so to the test ? Something along the lines of\n> \n> diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> index a67aac41b273..e24d10bd327c 100644\n> --- a/test/v4l2_compat/meson.build\n> +++ b/test/v4l2_compat/meson.build\n> @@ -1,10 +1,9 @@\n>  # SPDX-License-Identifier: CC0-1.0\n> \n> -pymod = import('python')\n> -py3 = pymod.find_installation('python3')\n> +if is_variable('v4l2_compat')\n> +    v4l2_compat_test = files('v4l2_compat_test.py')\n> \n> -v4l2_compat_test = files('v4l2_compat_test.py')\n> -\n> -test('v4l2_compat_test', py3,\n> -     args : v4l2_compat_test,\n> -     suite : 'v4l2_compat')\n> +    test('v4l2_compat_test', v4l2_compat_test,\n> +         args : v4l2_compat,\n> +         suite : 'v4l2_compat')\n> +endif\n> \n> > > +\n> > > +\n> > > +def grep(exp, arr):\n> > > +    return [s for s in arr if re.search(exp, s)]\n> > > +\n> > > +\n> > > +def run_with_stdout(cmd, args=\"\", env={}):\n> > > +    try:\n> > > +        with open(os.devnull, 'w') as devnull:\n> > > +            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> \n> Do you need to run this through a shell ? On my system, one of the video\n> nodes related to my integrated UVC webcam isn't handled by libcamera.\n> When v4l2-compliance is run on it with LD_PRELOAD=v4l2-compat.so, it\n> hangs if run through a shell, and doesn't otherwise. I suspect it's the\n> shell hanging, but if we don't need shell=True, then we may skip the\n> investigation :-)\n> \n> You will need to turn the first argument into an array. I've tried the\n> following, it seems to work.\n> \n> diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> index f56db4e91890..d05b3235a78c 100755\n> --- a/test/v4l2_compat/v4l2_compat_test.py\n> +++ b/test/v4l2_compat/v4l2_compat_test.py\n> @@ -33,10 +33,10 @@ def grep(exp, arr):\n>      return [s for s in arr if re.search(exp, s)]\n>  \n>  \n> -def run_with_stdout(cmd, args=\"\", env={}):\n> +def run_with_stdout(*args, env={}):\n>      try:\n>          with open(os.devnull, 'w') as devnull:\n> -            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> +            output = subprocess.check_output(args, env=env, stderr=devnull)\n>      except subprocess.CalledProcessError as err:\n>          output = err.output\n>      return output.decode(\"utf-8\").split(\"\\n\")\n> @@ -59,7 +59,7 @@ def print_output_arr(output_arr):\n>  \n>  \n>  def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> -    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> +    output = run_with_stdout(v4l2_compliance, \"-s\", \"-d\", device, env={\"LD_PRELOAD\": v4l2_compat})\n>      result = extract_result(output[-2])\n>      if result[\"driver\"] != \"libcamera\":\n>          return TestSkip\n> @@ -104,7 +104,7 @@ def main():\n>  \n>      failed = []\n>      for device in dev_nodes:\n> -        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> +        out = run_with_stdout(v4l2_ctl, \"-D\", \"-d\", device)\n>          driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n>          if driver not in supported_pipelines:\n>              continue\n> \n> And after further testing, it seems the test stills hangs from time to\n> time :-S I'll let you investigate if you can reproduce the issue.\n> \n> I'm also getting a floating point exception when I run v4l2-compliance\n> with LD_PRELOAD on my UVC webcam. Not too surprising as v4l2-compat\n> reports\n> \n> WARN V4L2Compat v4l2_camera_proxy.cpp:515 sizeimage of at least one format is zero. Streaming this format will cause a floating point exception.\n> \n> The camera produces formats::R8, which isn't listed in pixelFormatInfo.\n> I think that will be fixed when moving pixelFormatInfo out of the\n> v4l2-compat layer, like we've discussed offline.\n> \n> > > +    except subprocess.CalledProcessError as err:\n> > > +        output = err.output\n> > > +    return output.decode(\"utf-8\").split(\"\\n\")\n> > > +\n> > > +\n> > > +def extract_result(result):\n> > > +    res = result.split(\", \")\n> > > +    ret = {}\n> > > +    ret[\"total\"]     = int(res[0].split(\": \")[-1])\n> > > +    ret[\"succeeded\"] = int(res[1].split(\": \")[-1])\n> > > +    ret[\"failed\"]    = int(res[2].split(\": \")[-1])\n> > > +    ret[\"warnings\"]  = int(res[3].split(\": \")[-1])\n> > > +    ret[\"device\"]    = res[0].split()[4].strip(\":\")\n> > > +    ret[\"driver\"]    = res[0].split()[2]\n> > > +    return ret\n> > > +\n> > > +\n> > > +def print_output_arr(output_arr):\n> > > +    print(\"\\n\".join(output_arr))\n> > > +\n> > > +\n> > > +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> > > +    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> > > +    result = extract_result(output[-2])\n> > > +    if result[\"driver\"] != \"libcamera\":\n> > > +        return TestSkip\n> > > +\n> > > +    if result['failed'] == 0:\n> > > +        return TestPass\n> > > +\n> > > +    # vimc will fail s_fmt because it only supports framesizes that are\n> > > +    # multiples of 3\n> > > +    if base_driver == \"vimc\" and result[\"failed\"] <= 1:\n> > > +        failures = grep(\"fail\", output)\n> > > +        if re.search(\"S_FMT cannot handle an invalid format\", failures[0]) is None:\n> \n> Is there a reason not to use your grep function ?\n> \n> > > +            print_output_arr(output)\n> > > +            return TestFail\n> > > +        return TestPass\n> > > +\n> > > +    print_output_arr(output)\n> > > +    return TestFail\n> > > +\n> > > +\n> > > +def main():\n> > > +    v4l2_compliance = which(\"v4l2-compliance\")\n> > > +    if v4l2_compliance is None:\n> > > +        print(\"v4l2-compliance is not available\")\n> > > +        return TestSkip\n> > > +\n> > > +    v4l2_ctl = which(\"v4l2-ctl\")\n> > > +    if v4l2_ctl is None:\n> > > +        print(\"v4l2-ctl is not available\")\n> > > +        return TestSkip\n> > > +\n> > > +    v4l2_compat = find_file(\"v4l2-compat.so\", \".\")\n> > > +    if v4l2_compat is None:\n> > > +        print(\"v4l2-compat.so is not built\")\n> > > +        return TestSkip\n> > > +\n> > > +    dev_nodes = grep(\"video\", os.listdir(\"/dev\"))\n> \n> import glob\n> \n>     dev_nodes = glob.glob(\"/dev/video*\")\n> \n> > > +    dev_nodes = list(map(lambda s: f\"/dev/{s}\", dev_nodes))\n> \n> Or the more Pythonic syntax:\n> \n>     dev_nodes = [f\"/dev/{s]\" for s in dev_nodes]\n> \n> > > +    if len(dev_nodes) == 0:\n> > > +        print(\"no video nodes available to test with\")\n> > > +        return TestSkip\n> > > +\n> > > +    failed = []\n> > > +    for device in dev_nodes:\n> > > +        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> > > +        driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n> > > +        if driver not in supported_pipelines:\n> > > +            continue\n> > > +\n> > > +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)\n> > > +        if ret == TestFail:\n> > > +            failed.append(device)\n> > > +\n> > > +    if len(failed) > 0:\n> > > +        print(f\"Failed {len(failed)} tests:\")\n> > > +        for device in failed:\n> > > +            print(f\"- {device}\")\n> > > +\n> > > +    return TestPass if not failed else TestFail\n> > > +\n> > > +\n> > > +if __name__ == '__main__':\n> > > +    sys.exit(main())\n> \n> -- \n> Regards,\n> \n> Laurent Pinchart","headers":{"Return-Path":"<niklas.soderlund@ragnatech.se>","Received":["from mail-lj1-x243.google.com (mail-lj1-x243.google.com\n\t[IPv6:2a00:1450:4864:20::243])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id ED27B603BB\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 25 Jun 2020 00:36:32 +0200 (CEST)","by mail-lj1-x243.google.com with SMTP id i27so4322720ljb.12\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 24 Jun 2020 15:36:32 -0700 (PDT)","from localhost (h-209-203.A463.priv.bahnhof.se. [155.4.209.203])\n\tby smtp.gmail.com with ESMTPSA id\n\ti15sm5532888lfl.57.2020.06.24.15.36.31\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tWed, 24 Jun 2020 15:36:31 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected)\n\theader.d=ragnatech-se.20150623.gappssmtp.com\n\theader.i=@ragnatech-se.20150623.gappssmtp.com header.b=\"Az+NapXr\"; \n\tdkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=ragnatech-se.20150623.gappssmtp.com; s=20150623;\n\th=date:from:to:cc:subject:message-id:references:mime-version\n\t:content-disposition:content-transfer-encoding:in-reply-to;\n\tbh=nxWE54qOrsBFvvGMqMI4PKKWFPpRxQ6Ig0KXAnAvglA=;\n\tb=Az+NapXreDg6v+mjJ0MP8v4otEvKM50szUNQEAFUYcYnF/+5GTKjvRiMnN88LNn3wC\n\tt+gISdqz+pZiu4OYtVYTZdKmo46sV0fldZc+/qXTnjj2yqtF59nQl4d+1OIaN5IoDtb5\n\tZpmbYkogtd6aziXrgIs89Jn2i16JV87rSCtH9D1gUVnDKms3vIKsvr1Spp1wvBXumF9y\n\tBMCWqHoPrMiFdLUlYOa1VlBmCyc4oBMYVYX4lMqmZabn3jCjn1+WDqvEijn01Sc+VWK5\n\tTUduojLOmMKqj6+sqB5ztl/f7dt2CjXCrjYHszmx6MV+VKZx4jU7bmq2GXX1OQbXaL17\n\tEiIg==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:date:from:to:cc:subject:message-id:references\n\t:mime-version:content-disposition:content-transfer-encoding\n\t:in-reply-to;\n\tbh=nxWE54qOrsBFvvGMqMI4PKKWFPpRxQ6Ig0KXAnAvglA=;\n\tb=s0tsCht1M4M00hPvexnW25+JE/TKIETGva1Dj2GzBgnOFHKyjMW94GdGUt/ogj3TAY\n\t/Fa17U8GKS8SlH35DdNKJfgjDBRsrNq5rpL3gGmCqOM9xhc1/mq7xtPYiUeba/a3K6iv\n\tAbgfmZ207jEhmajY1t0Ti+T9xPzPFBen8AQTfRXU/sfyBWUhqNMPGLiq29UL11tDpQno\n\t6/thNnmPTupEfjO5NS6f601yz081cPGb/pK94yhf5IeaiAtO55u13nSIjl8XQUrYHEjP\n\tWZgwBP4X/wC7Gjxvuj9GZAv2+D3qkQQYZ9zl0OyhoL+8+yF/m1Fr0rpz4jgnx40LyfYQ\n\t6/Sg==","X-Gm-Message-State":"AOAM5338i5ec/EQN9BumIdkJcTgtz6p1j2B28URYFJjoUqdsZRhC5x9z\n\t5e5uol3tLUR2anP36TjSfCe1xMTaArI=","X-Google-Smtp-Source":"ABdhPJy1dya4O6IkVJVKLbd0L/YUNrxU03g86MmBZ9J7Pa+/n0e8R9ZkLcw8z6sICubVMfATV/LXwQ==","X-Received":"by 2002:a05:651c:3c2:: with SMTP id\n\tf2mr4123858ljp.37.1593038192008; \n\tWed, 24 Jun 2020 15:36:32 -0700 (PDT)","Date":"Thu, 25 Jun 2020 00:36:30 +0200","From":"Niklas =?iso-8859-1?q?S=F6derlund?= <niklas.soderlund@ragnatech.se>","To":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","Cc":"Paul Elder <paul.elder@ideasonboard.com>,\n\tlibcamera-devel@lists.libcamera.org","Message-ID":"<20200624223630.GA10256@oden.dyn.berto.se>","References":"<20200624140752.46066-1-paul.elder@ideasonboard.com>\n\t<20200624173804.GC1595450@oden.dyn.berto.se>\n\t<20200624222641.GI5980@pendragon.ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=iso-8859-1","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"<20200624222641.GI5980@pendragon.ideasonboard.com>","Subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","X-List-Received-Date":"Wed, 24 Jun 2020 22:36:33 -0000"}},{"id":10883,"web_url":"https://patchwork.libcamera.org/comment/10883/","msgid":"<20200626075144.GA108994@pyrite.rasen.tech>","date":"2020-06-26T07:51:44","subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"Hi Niklas,\n\nThank you for the comments.\n\nOn Wed, Jun 24, 2020 at 07:38:04PM +0200, Niklas Söderlund wrote:\n> Hi Paul,\n> \n> Thanks for your work.\n> \n> On 2020-06-24 23:07:52 +0900, Paul Elder wrote:\n> > Test the V4L2 compatibility layer by running v4l2-compliance -s on every\n> > /dev/video* device.\n> \n> I wonder if this is not a tad to aggressive. Imagine the test being run \n> on a device where one video node is not covered by a libcamera pipeline \n> and also does not pass v4l2-compliance. Would this not lead to the \n> libcamera test suite to fail?\n> \n> Would it be possible you think to check the driver name of the video \n> device and check it against an whitelist (vimc, vivid, ?) before running \n> the v4l2-compliance suite?\n> \n> You can get the driver name from\n> \n>     v4l2-ctl -D -d /dev/videoX\n\nYes, as Laurent pointed out, I do do this already :)\n\nThere was one part though, that I ran v4l2-compliance -s and then\nchecked if libcamera actually intercepted it... in v2 I've moved this\nover to v4l2-ctl.\n\n\nThanks,\n\nPaul\n\n> > \n> > Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>\n> > ---\n> >  test/meson.build                     |   1 +\n> >  test/v4l2_compat/meson.build         |  10 +++\n> >  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++\n> >  3 files changed, 136 insertions(+)\n> >  create mode 100644 test/v4l2_compat/meson.build\n> >  create mode 100755 test/v4l2_compat/v4l2_compat_test.py\n> > \n> > diff --git a/test/meson.build b/test/meson.build\n> > index a868813..591920f 100644\n> > --- a/test/meson.build\n> > +++ b/test/meson.build\n> > @@ -12,6 +12,7 @@ subdir('pipeline')\n> >  subdir('process')\n> >  subdir('serialization')\n> >  subdir('stream')\n> > +subdir('v4l2_compat')\n> >  subdir('v4l2_subdevice')\n> >  subdir('v4l2_videodevice')\n> >  \n> > diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> > new file mode 100644\n> > index 0000000..a67aac4\n> > --- /dev/null\n> > +++ b/test/v4l2_compat/meson.build\n> > @@ -0,0 +1,10 @@\n> > +# SPDX-License-Identifier: CC0-1.0\n> > +\n> > +pymod = import('python')\n> > +py3 = pymod.find_installation('python3')\n> > +\n> > +v4l2_compat_test = files('v4l2_compat_test.py')\n> > +\n> > +test('v4l2_compat_test', py3,\n> > +     args : v4l2_compat_test,\n> > +     suite : 'v4l2_compat')\n> > diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> > new file mode 100755\n> > index 0000000..f56db4e\n> > --- /dev/null\n> > +++ b/test/v4l2_compat/v4l2_compat_test.py\n> > @@ -0,0 +1,125 @@\n> > +#!/usr/bin/env python3\n> > +# SPDX-License-Identifier: GPL-2.0-or-later\n> > +# Copyright (C) 2019, Google Inc.\n> > +#\n> > +# Author: Paul Elder <paul.elder@ideasonboard.com>\n> > +#\n> > +# v4l2_compat_test.py - Test the V4L2 compatibility layer\n> > +\n> > +import os\n> > +import re\n> > +from shutil import which\n> > +import subprocess\n> > +import sys\n> > +\n> > +TestPass = 0\n> > +TestFail = -1\n> > +TestSkip = 77\n> > +\n> > +\n> > +supported_pipelines = [\n> > +    \"uvcvideo\",\n> > +    \"vimc\",\n> > +]\n> > +\n> > +\n> > +def find_file(name, path):\n> > +    for root, dirs, files in os.walk(path):\n> > +        if name in files:\n> > +            return os.path.join(root, name)\n> > +\n> > +\n> > +def grep(exp, arr):\n> > +    return [s for s in arr if re.search(exp, s)]\n> > +\n> > +\n> > +def run_with_stdout(cmd, args=\"\", env={}):\n> > +    try:\n> > +        with open(os.devnull, 'w') as devnull:\n> > +            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> > +    except subprocess.CalledProcessError as err:\n> > +        output = err.output\n> > +    return output.decode(\"utf-8\").split(\"\\n\")\n> > +\n> > +\n> > +def extract_result(result):\n> > +    res = result.split(\", \")\n> > +    ret = {}\n> > +    ret[\"total\"]     = int(res[0].split(\": \")[-1])\n> > +    ret[\"succeeded\"] = int(res[1].split(\": \")[-1])\n> > +    ret[\"failed\"]    = int(res[2].split(\": \")[-1])\n> > +    ret[\"warnings\"]  = int(res[3].split(\": \")[-1])\n> > +    ret[\"device\"]    = res[0].split()[4].strip(\":\")\n> > +    ret[\"driver\"]    = res[0].split()[2]\n> > +    return ret\n> > +\n> > +\n> > +def print_output_arr(output_arr):\n> > +    print(\"\\n\".join(output_arr))\n> > +\n> > +\n> > +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> > +    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> > +    result = extract_result(output[-2])\n> > +    if result[\"driver\"] != \"libcamera\":\n> > +        return TestSkip\n> > +\n> > +    if result['failed'] == 0:\n> > +        return TestPass\n> > +\n> > +    # vimc will fail s_fmt because it only supports framesizes that are\n> > +    # multiples of 3\n> > +    if base_driver == \"vimc\" and result[\"failed\"] <= 1:\n> > +        failures = grep(\"fail\", output)\n> > +        if re.search(\"S_FMT cannot handle an invalid format\", failures[0]) is None:\n> > +            print_output_arr(output)\n> > +            return TestFail\n> > +        return TestPass\n> > +\n> > +    print_output_arr(output)\n> > +    return TestFail\n> > +\n> > +\n> > +def main():\n> > +    v4l2_compliance = which(\"v4l2-compliance\")\n> > +    if v4l2_compliance is None:\n> > +        print(\"v4l2-compliance is not available\")\n> > +        return TestSkip\n> > +\n> > +    v4l2_ctl = which(\"v4l2-ctl\")\n> > +    if v4l2_ctl is None:\n> > +        print(\"v4l2-ctl is not available\")\n> > +        return TestSkip\n> > +\n> > +    v4l2_compat = find_file(\"v4l2-compat.so\", \".\")\n> > +    if v4l2_compat is None:\n> > +        print(\"v4l2-compat.so is not built\")\n> > +        return TestSkip\n> > +\n> > +    dev_nodes = grep(\"video\", os.listdir(\"/dev\"))\n> > +    dev_nodes = list(map(lambda s: f\"/dev/{s}\", dev_nodes))\n> > +    if len(dev_nodes) == 0:\n> > +        print(\"no video nodes available to test with\")\n> > +        return TestSkip\n> > +\n> > +    failed = []\n> > +    for device in dev_nodes:\n> > +        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> > +        driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n> > +        if driver not in supported_pipelines:\n> > +            continue\n> > +\n> > +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)\n> > +        if ret == TestFail:\n> > +            failed.append(device)\n> > +\n> > +    if len(failed) > 0:\n> > +        print(f\"Failed {len(failed)} tests:\")\n> > +        for device in failed:\n> > +            print(f\"- {device}\")\n> > +\n> > +    return TestPass if not failed else TestFail\n> > +\n> > +\n> > +if __name__ == '__main__':\n> > +    sys.exit(main())\n> > -- \n> > 2.27.0\n> > \n> > _______________________________________________\n> > libcamera-devel mailing list\n> > libcamera-devel@lists.libcamera.org\n> > https://lists.libcamera.org/listinfo/libcamera-devel\n> \n> -- \n> Regards,\n> Niklas Söderlund","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6ABE1C0109\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 26 Jun 2020 07:51:54 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 032C4609C6;\n\tFri, 26 Jun 2020 09:51:54 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 85930603B6\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 26 Jun 2020 09:51:53 +0200 (CEST)","from pyrite.rasen.tech (unknown\n\t[IPv6:2400:4051:61:600:2c71:1b79:d06d:5032])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id D93BA72E;\n\tFri, 26 Jun 2020 09:51:51 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"Q5JdxiGX\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1593157913;\n\tbh=u2aIT1SNgZJk9fc9jTiOJUqMGH9esB4JcpHBMg8+Gys=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=Q5JdxiGX4nRSC1mho3L+Cyvo9jyy2jwxQC8wsmD4sVH7C0k7Ppsf3eMyUJ+Q1Frxi\n\tgaV32QYw9BbP1j/jK49BF+qGNeXSGqPVWc3vKvn339IXF+rEnGpHYIQYJ5xnQC269D\n\tJ9uGOgJ8D17ze9a1fIVOm+Ovv4KyWFrJRKSLuNu0=","Date":"Fri, 26 Jun 2020 16:51:44 +0900","From":"Paul Elder <paul.elder@ideasonboard.com>","To":"Niklas =?iso-8859-1?q?S=F6derlund?= <niklas.soderlund@ragnatech.se>","Message-ID":"<20200626075144.GA108994@pyrite.rasen.tech>","References":"<20200624140752.46066-1-paul.elder@ideasonboard.com>\n\t<20200624173804.GC1595450@oden.dyn.berto.se>","MIME-Version":"1.0","Content-Disposition":"inline","In-Reply-To":"<20200624173804.GC1595450@oden.dyn.berto.se>","Subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"iso-8859-1\"","Content-Transfer-Encoding":"quoted-printable","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}},{"id":10884,"web_url":"https://patchwork.libcamera.org/comment/10884/","msgid":"<20200626075737.GB108994@pyrite.rasen.tech>","date":"2020-06-26T07:57:37","subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"Hi Laurent,\n\nThank you for the review.\n\nOn Thu, Jun 25, 2020 at 01:26:41AM +0300, Laurent Pinchart wrote:\n> Hi Niklas and Paul,\n> \n> On Wed, Jun 24, 2020 at 07:38:04PM +0200, Niklas Söderlund wrote:\n> > Hi Paul,\n> > \n> > Thanks for your work.\n> > \n> > On 2020-06-24 23:07:52 +0900, Paul Elder wrote:\n> > > Test the V4L2 compatibility layer by running v4l2-compliance -s on every\n> > > /dev/video* device.\n> > \n> > I wonder if this is not a tad to aggressive. Imagine the test being run \n> > on a device where one video node is not covered by a libcamera pipeline \n> > and also does not pass v4l2-compliance. Would this not lead to the \n> > libcamera test suite to fail?\n> > \n> > Would it be possible you think to check the driver name of the video \n> > device and check it against an whitelist (vimc, vivid, ?) before running \n> > the v4l2-compliance suite?\n> > \n> > You can get the driver name from\n> > \n> >     v4l2-ctl -D -d /dev/videoX\n> \n> Isn't this exactly what the code below is doing ? :-)\n> \n> > > Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>\n> > > ---\n> > >  test/meson.build                     |   1 +\n> > >  test/v4l2_compat/meson.build         |  10 +++\n> > >  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++\n> > >  3 files changed, 136 insertions(+)\n> > >  create mode 100644 test/v4l2_compat/meson.build\n> > >  create mode 100755 test/v4l2_compat/v4l2_compat_test.py\n> > > \n> > > diff --git a/test/meson.build b/test/meson.build\n> > > index a868813..591920f 100644\n> > > --- a/test/meson.build\n> > > +++ b/test/meson.build\n> > > @@ -12,6 +12,7 @@ subdir('pipeline')\n> > >  subdir('process')\n> > >  subdir('serialization')\n> > >  subdir('stream')\n> > > +subdir('v4l2_compat')\n> > >  subdir('v4l2_subdevice')\n> > >  subdir('v4l2_videodevice')\n> > >  \n> > > diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> > > new file mode 100644\n> > > index 0000000..a67aac4\n> > > --- /dev/null\n> > > +++ b/test/v4l2_compat/meson.build\n> > > @@ -0,0 +1,10 @@\n> > > +# SPDX-License-Identifier: CC0-1.0\n> > > +\n> > > +pymod = import('python')\n> > > +py3 = pymod.find_installation('python3')\n> > > +\n> > > +v4l2_compat_test = files('v4l2_compat_test.py')\n> > > +\n> > > +test('v4l2_compat_test', py3,\n> > > +     args : v4l2_compat_test,\n> > > +     suite : 'v4l2_compat')\n> \n> Would this be simpler ?\n> \n> test('v4l2_compat_test', v4l2_compat_test,\n>      suite : 'v4l2_compat')\n\nAh yes, that is much better. I wasn't sure how tests with python worked\nin meson.\n\n> > > diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> > > new file mode 100755\n> > > index 0000000..f56db4e\n> > > --- /dev/null\n> > > +++ b/test/v4l2_compat/v4l2_compat_test.py\n> > > @@ -0,0 +1,125 @@\n> > > +#!/usr/bin/env python3\n> > > +# SPDX-License-Identifier: GPL-2.0-or-later\n> > > +# Copyright (C) 2019, Google Inc.\n> \n> 2020 ?\n\nYes :)\n\n> > > +#\n> > > +# Author: Paul Elder <paul.elder@ideasonboard.com>\n> > > +#\n> > > +# v4l2_compat_test.py - Test the V4L2 compatibility layer\n> > > +\n> > > +import os\n> > > +import re\n> > > +from shutil import which\n> \n> I'd just import shutil and use shutil.which, up to you.\n> \n> > > +import subprocess\n> > > +import sys\n> > > +\n> > > +TestPass = 0\n> > > +TestFail = -1\n> > > +TestSkip = 77\n> > > +\n> > > +\n> > > +supported_pipelines = [\n> > > +    \"uvcvideo\",\n> \n> Our python scripts usually use single quotes for strings.\n> \n> > > +    \"vimc\",\n> > > +]\n> > > +\n> > > +\n> > > +def find_file(name, path):\n> > > +    for root, dirs, files in os.walk(path):\n> > > +        if name in files:\n> > > +            return os.path.join(root, name)\n> \n> That's a bit of heavy artillery. How about passing the path to\n> v4l2-compat.so to the test ? Something along the lines of\n\nAh yes, much better.\n\n> diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build\n> index a67aac41b273..e24d10bd327c 100644\n> --- a/test/v4l2_compat/meson.build\n> +++ b/test/v4l2_compat/meson.build\n> @@ -1,10 +1,9 @@\n>  # SPDX-License-Identifier: CC0-1.0\n> \n> -pymod = import('python')\n> -py3 = pymod.find_installation('python3')\n> +if is_variable('v4l2_compat')\n> +    v4l2_compat_test = files('v4l2_compat_test.py')\n> \n> -v4l2_compat_test = files('v4l2_compat_test.py')\n> -\n> -test('v4l2_compat_test', py3,\n> -     args : v4l2_compat_test,\n> -     suite : 'v4l2_compat')\n> +    test('v4l2_compat_test', v4l2_compat_test,\n> +         args : v4l2_compat,\n> +         suite : 'v4l2_compat')\n> +endif\n> \n> > > +\n> > > +\n> > > +def grep(exp, arr):\n> > > +    return [s for s in arr if re.search(exp, s)]\n> > > +\n> > > +\n> > > +def run_with_stdout(cmd, args=\"\", env={}):\n> > > +    try:\n> > > +        with open(os.devnull, 'w') as devnull:\n> > > +            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> \n> Do you need to run this through a shell ? On my system, one of the video\n> nodes related to my integrated UVC webcam isn't handled by libcamera.\n> When v4l2-compliance is run on it with LD_PRELOAD=v4l2-compat.so, it\n> hangs if run through a shell, and doesn't otherwise. I suspect it's the\n> shell hanging, but if we don't need shell=True, then we may skip the\n> investigation :-)\n\nIt looks like we don't need to run it through as shell, so I've dropped\nit.\n\n> You will need to turn the first argument into an array. I've tried the\n> following, it seems to work.\n> \n> diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py\n> index f56db4e91890..d05b3235a78c 100755\n> --- a/test/v4l2_compat/v4l2_compat_test.py\n> +++ b/test/v4l2_compat/v4l2_compat_test.py\n> @@ -33,10 +33,10 @@ def grep(exp, arr):\n>      return [s for s in arr if re.search(exp, s)]\n>  \n>  \n> -def run_with_stdout(cmd, args=\"\", env={}):\n> +def run_with_stdout(*args, env={}):\n>      try:\n>          with open(os.devnull, 'w') as devnull:\n> -            output = subprocess.check_output(f\"{cmd} {args}\", shell=True, env=env, stderr=devnull)\n> +            output = subprocess.check_output(args, env=env, stderr=devnull)\n>      except subprocess.CalledProcessError as err:\n>          output = err.output\n>      return output.decode(\"utf-8\").split(\"\\n\")\n> @@ -59,7 +59,7 @@ def print_output_arr(output_arr):\n>  \n>  \n>  def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> -    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> +    output = run_with_stdout(v4l2_compliance, \"-s\", \"-d\", device, env={\"LD_PRELOAD\": v4l2_compat})\n>      result = extract_result(output[-2])\n>      if result[\"driver\"] != \"libcamera\":\n>          return TestSkip\n> @@ -104,7 +104,7 @@ def main():\n>  \n>      failed = []\n>      for device in dev_nodes:\n> -        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> +        out = run_with_stdout(v4l2_ctl, \"-D\", \"-d\", device)\n>          driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n>          if driver not in supported_pipelines:\n>              continue\n> \n> And after further testing, it seems the test stills hangs from time to\n> time :-S I'll let you investigate if you can reproduce the issue.\n> \n> I'm also getting a floating point exception when I run v4l2-compliance\n> with LD_PRELOAD on my UVC webcam. Not too surprising as v4l2-compat\n> reports\n> \n> WARN V4L2Compat v4l2_camera_proxy.cpp:515 sizeimage of at least one format is zero. Streaming this format will cause a floating point exception.\n> \n> The camera produces formats::R8, which isn't listed in pixelFormatInfo.\n> I think that will be fixed when moving pixelFormatInfo out of the\n> v4l2-compat layer, like we've discussed offline.\n\nYeah, this has to be fixed.\n\n> > > +    except subprocess.CalledProcessError as err:\n> > > +        output = err.output\n> > > +    return output.decode(\"utf-8\").split(\"\\n\")\n> > > +\n> > > +\n> > > +def extract_result(result):\n> > > +    res = result.split(\", \")\n> > > +    ret = {}\n> > > +    ret[\"total\"]     = int(res[0].split(\": \")[-1])\n> > > +    ret[\"succeeded\"] = int(res[1].split(\": \")[-1])\n> > > +    ret[\"failed\"]    = int(res[2].split(\": \")[-1])\n> > > +    ret[\"warnings\"]  = int(res[3].split(\": \")[-1])\n> > > +    ret[\"device\"]    = res[0].split()[4].strip(\":\")\n> > > +    ret[\"driver\"]    = res[0].split()[2]\n> > > +    return ret\n> > > +\n> > > +\n> > > +def print_output_arr(output_arr):\n> > > +    print(\"\\n\".join(output_arr))\n> > > +\n> > > +\n> > > +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):\n> > > +    output = run_with_stdout(v4l2_compliance, f\"-s -d {device}\", {\"LD_PRELOAD\": v4l2_compat})\n> > > +    result = extract_result(output[-2])\n> > > +    if result[\"driver\"] != \"libcamera\":\n> > > +        return TestSkip\n> > > +\n> > > +    if result['failed'] == 0:\n> > > +        return TestPass\n> > > +\n> > > +    # vimc will fail s_fmt because it only supports framesizes that are\n> > > +    # multiples of 3\n> > > +    if base_driver == \"vimc\" and result[\"failed\"] <= 1:\n> > > +        failures = grep(\"fail\", output)\n> > > +        if re.search(\"S_FMT cannot handle an invalid format\", failures[0]) is None:\n> \n> Is there a reason not to use your grep function ?\n\nMy grep function greps for a substring within a list of strings.\n\nre.search(\"S_FMT cannot handle an invalid format\", failures[0])\nvs\ngrep(\"S_FMT cannot handle an invalid format\", [failures[0]])\n\nI think re.search is good enough for this.\n\n> > > +            print_output_arr(output)\n> > > +            return TestFail\n> > > +        return TestPass\n> > > +\n> > > +    print_output_arr(output)\n> > > +    return TestFail\n> > > +\n> > > +\n> > > +def main():\n> > > +    v4l2_compliance = which(\"v4l2-compliance\")\n> > > +    if v4l2_compliance is None:\n> > > +        print(\"v4l2-compliance is not available\")\n> > > +        return TestSkip\n> > > +\n> > > +    v4l2_ctl = which(\"v4l2-ctl\")\n> > > +    if v4l2_ctl is None:\n> > > +        print(\"v4l2-ctl is not available\")\n> > > +        return TestSkip\n> > > +\n> > > +    v4l2_compat = find_file(\"v4l2-compat.so\", \".\")\n> > > +    if v4l2_compat is None:\n> > > +        print(\"v4l2-compat.so is not built\")\n> > > +        return TestSkip\n> > > +\n> > > +    dev_nodes = grep(\"video\", os.listdir(\"/dev\"))\n> \n> import glob\n> \n>     dev_nodes = glob.glob(\"/dev/video*\")\n>\n> > > +    dev_nodes = list(map(lambda s: f\"/dev/{s}\", dev_nodes))\n> \n> Or the more Pythonic syntax:\n> \n>     dev_nodes = [f\"/dev/{s]\" for s in dev_nodes]\n\nWith glob, this is no longer necessary :)\n\n> > > +    if len(dev_nodes) == 0:\n> > > +        print(\"no video nodes available to test with\")\n> > > +        return TestSkip\n> > > +\n> > > +    failed = []\n> > > +    for device in dev_nodes:\n> > > +        out = run_with_stdout(v4l2_ctl, f\"-D -d {device}\")\n> > > +        driver = grep(\"Driver name\", out)[0].split(\":\")[-1].strip()\n> > > +        if driver not in supported_pipelines:\n> > > +            continue\n> > > +\n> > > +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)\n> > > +        if ret == TestFail:\n> > > +            failed.append(device)\n> > > +\n> > > +    if len(failed) > 0:\n> > > +        print(f\"Failed {len(failed)} tests:\")\n> > > +        for device in failed:\n> > > +            print(f\"- {device}\")\n> > > +\n> > > +    return TestPass if not failed else TestFail\n> > > +\n> > > +\n> > > +if __name__ == '__main__':\n> > > +    sys.exit(main())\n\nI've also figured out how to increase the test timeout in meson.\nv2 coming up!\n\n\nThanks,\n\nPaul","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 60E99C2E65\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 26 Jun 2020 07:57:47 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C57BB609C6;\n\tFri, 26 Jun 2020 09:57:46 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 659E7603B6\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 26 Jun 2020 09:57:45 +0200 (CEST)","from pyrite.rasen.tech (unknown\n\t[IPv6:2400:4051:61:600:2c71:1b79:d06d:5032])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 81AE372E;\n\tFri, 26 Jun 2020 09:57:43 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"JtMlSBk3\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1593158265;\n\tbh=sGWlUWO/gk9YfXuis/l1I+ntp9i32KxNLMbjVfRDW4k=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=JtMlSBk3fhPECDyFpK9R0/Nq6brAyjNU9ygvDKUVT1pcNOywv75jPCh0lutys+VCC\n\tbH7b1X770M3syRfs8oNLjiWh2UJSoX8MxgXqFKCf58mgMSFwvqfQlEhyS6IXOvtE9J\n\tqlnABUBd4v8Q9YG+j/AEJoQZARzx1MxmcUSlZfns=","Date":"Fri, 26 Jun 2020 16:57:37 +0900","From":"Paul Elder <paul.elder@ideasonboard.com>","To":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","Message-ID":"<20200626075737.GB108994@pyrite.rasen.tech>","References":"<20200624140752.46066-1-paul.elder@ideasonboard.com>\n\t<20200624173804.GC1595450@oden.dyn.berto.se>\n\t<20200624222641.GI5980@pendragon.ideasonboard.com>","MIME-Version":"1.0","Content-Disposition":"inline","In-Reply-To":"<20200624222641.GI5980@pendragon.ideasonboard.com>","Subject":"Re: [libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for\n\tv4l2_compat","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"iso-8859-1\"","Content-Transfer-Encoding":"quoted-printable","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]