From patchwork Thu Jul 9 13:28:35 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 8738 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 83816BD792 for ; Thu, 9 Jul 2020 13:29:48 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 51121611BA; Thu, 9 Jul 2020 15:29:48 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="qVeLRk4l"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id C163E611B2 for ; Thu, 9 Jul 2020 15:29:46 +0200 (CEST) Received: from pyrite.rasen.tech (unknown [IPv6:2400:4051:61:600:2c71:1b79:d06d:5032]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id EB49E525; Thu, 9 Jul 2020 15:29:44 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1594301386; bh=snaqUun6n9obVuM8zCpQZ8/093P7lRU+gTulHWcuI/8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=qVeLRk4lVqnYuV1ycvCVXUsPumnw+fDk795OjRTeswd/h9cDyK0V7rSmSQLScqMCG aYFoHzvcdKUoOHw64prleQscXi6lmjeAuCKUhpMzZQACvyl/WlxypvZaxGhvdtzXK7 CKdJ6E5CfukhhnYqsbJ7VrqZkFfuygU95Vp3P0O8= From: Paul Elder To: libcamera-devel@lists.libcamera.org Date: Thu, 9 Jul 2020 22:28:35 +0900 Message-Id: <20200709132835.112593-24-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200709132835.112593-1-paul.elder@ideasonboard.com> References: <20200709132835.112593-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 23/23] tests: v4l2_compat: Add test for v4l2_compat X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Test the V4L2 compatibility layer by running v4l2-compliance -s on every /dev/video* device. Signed-off-by: Paul Elder Reviewed-by: Laurent Pinchart --- Changes in v5 - add raspi to supported pipelines Changes in v4: - fail the test if v4l2-ctl fails (for example, if libcamera dies fatally) Changes in v3: - use argparse - add argument to test all cameras, and test only one camera per driver by default - print the devices that we're testing and if each succeeded or failed Changes in v2: - change all strings to single-quote - simplify meson.build - get path to v4l2-compat.so from cli arg, rather than running custom find() - remove find_file() - extend test timeout to 60 seconds (from default of 30) - don't run v4l2-compliance subprocesses in shell - check if v4l2-compliance runs are killed by signal (eg. the known SIGABRT due to floating point exception) - move the check to see if libcamera supports the camera to v4l2-ctl instead of v4l2-compliance (in v1 we only checked if the pipeline was supported with v4l2-ctl) --- test/meson.build | 1 + test/v4l2_compat/meson.build | 10 ++ test/v4l2_compat/v4l2_compat_test.py | 152 +++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 test/v4l2_compat/meson.build create mode 100755 test/v4l2_compat/v4l2_compat_test.py diff --git a/test/meson.build b/test/meson.build index 7808a26..f41d6e7 100644 --- a/test/meson.build +++ b/test/meson.build @@ -12,6 +12,7 @@ subdir('pipeline') subdir('process') subdir('serialization') subdir('stream') +subdir('v4l2_compat') subdir('v4l2_subdevice') subdir('v4l2_videodevice') diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build new file mode 100644 index 0000000..5b29de7 --- /dev/null +++ b/test/v4l2_compat/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +if is_variable('v4l2_compat') + v4l2_compat_test = files('v4l2_compat_test.py') + + test('v4l2_compat_test', v4l2_compat_test, + args : v4l2_compat, + suite : 'v4l2_compat', + timeout : 60) +endif diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py new file mode 100755 index 0000000..99b3d5a --- /dev/null +++ b/test/v4l2_compat/v4l2_compat_test.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2020, Google Inc. +# +# Author: Paul Elder +# +# v4l2_compat_test.py - Test the V4L2 compatibility layer + +import argparse +import glob +import os +import re +import shutil +import signal +import subprocess +import sys + +TestPass = 0 +TestFail = -1 +TestSkip = 77 + + +supported_pipelines = [ + 'bcm2835-isp', + 'uvcvideo', + 'vimc', +] + + +def grep(exp, arr): + return [s for s in arr if re.search(exp, s)] + + +def run_with_stdout(*args, env={}): + try: + with open(os.devnull, 'w') as devnull: + output = subprocess.check_output(args, env=env, stderr=devnull) + ret = 0 + except subprocess.CalledProcessError as err: + output = err.output + ret = err.returncode + return ret, output.decode('utf-8').split('\n') + + +def extract_result(result): + res = result.split(', ') + ret = {} + ret['total'] = int(res[0].split(': ')[-1]) + ret['succeeded'] = int(res[1].split(': ')[-1]) + ret['failed'] = int(res[2].split(': ')[-1]) + ret['warnings'] = int(res[3].split(': ')[-1]) + ret['device'] = res[0].split()[4].strip(':') + ret['driver'] = res[0].split()[2] + return ret + + +def print_output_arr(output_arr): + print('\n'.join(output_arr)) + + +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver): + ret, output = run_with_stdout(v4l2_compliance, '-s', '-d', device, env={'LD_PRELOAD': v4l2_compat}) + if ret < 0: + print_output_arr(output) + print(f'Test for {device} terminated due to signal {signal.Signals(-ret).name}') + return TestFail + + result = extract_result(output[-2]) + if result['failed'] == 0: + return TestPass + + # vimc will fail s_fmt because it only supports framesizes that are + # multiples of 3 + if base_driver == 'vimc' and result['failed'] == 1: + failures = grep('fail', output) + if re.search('S_FMT cannot handle an invalid format', failures[0]) is None: + print_output_arr(output) + return TestFail + return TestPass + + print_output_arr(output) + return TestFail + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('-a', '--all', action='store_true', + help='Test all available cameras') + parser.add_argument('v4l2_compat', type=str, + help='Path to v4l2-compat.so') + args = parser.parse_args(argv[1:]) + + v4l2_compat = args.v4l2_compat + + v4l2_compliance = shutil.which('v4l2-compliance') + if v4l2_compliance is None: + print('v4l2-compliance is not available') + return TestSkip + + v4l2_ctl = shutil.which('v4l2-ctl') + if v4l2_ctl is None: + print('v4l2-ctl is not available') + return TestSkip + + dev_nodes = glob.glob('/dev/video*') + if len(dev_nodes) == 0: + print('no video nodes available to test with') + return TestSkip + + failed = [] + drivers_tested = {} + for device in dev_nodes: + ret, out = run_with_stdout(v4l2_ctl, '-D', '-d', device, env={'LD_PRELOAD': v4l2_compat}) + if ret < 0: + failed.append(device) + print(f'v4l2-ctl failed on {device} with v4l2-compat') + continue + driver = grep('Driver name', out)[0].split(':')[-1].strip() + if driver != "libcamera": + continue + + ret, out = run_with_stdout(v4l2_ctl, '-D', '-d', device) + if ret < 0: + failed.append(device) + print(f'v4l2-ctl failed on {device} without v4l2-compat') + continue + driver = grep('Driver name', out)[0].split(':')[-1].strip() + if driver not in supported_pipelines: + continue + + if not args.all and driver in drivers_tested: + continue + + print(f'Testing {device} with {driver} driver... ', end='') + ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver) + if ret == TestFail: + failed.append(device) + print('failed') + else: + print('success') + drivers_tested[driver] = True + + if len(failed) > 0: + print(f'Failed {len(failed)} tests:') + for device in failed: + print(f'- {device}') + + return TestPass if not failed else TestFail + + +if __name__ == '__main__': + sys.exit(main(sys.argv))