From patchwork Wed Mar 10 15:44:13 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 11543 X-Patchwork-Delegate: niklas.soderlund@ragnatech.se Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id DF880BD80C for ; Wed, 10 Mar 2021 15:44:27 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id AE59F602E6; Wed, 10 Mar 2021 16:44:27 +0100 (CET) Received: from vsp-unauthed02.binero.net (vsp-unauthed02.binero.net [195.74.38.227]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6DFA0602E5 for ; Wed, 10 Mar 2021 16:44:25 +0100 (CET) X-Halon-ID: 7ca7bd1f-81b7-11eb-b73f-0050569116f7 Authorized-sender: niklas.soderlund@fsdn.se Received: from bismarck.berto.se (p54ac5521.dip0.t-ipconnect.de [84.172.85.33]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 7ca7bd1f-81b7-11eb-b73f-0050569116f7; Wed, 10 Mar 2021 16:44:23 +0100 (CET) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Wed, 10 Mar 2021 16:44:13 +0100 Message-Id: <20210310154414.3560115-2-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.30.1 In-Reply-To: <20210310154414.3560115-1-niklas.soderlund@ragnatech.se> References: <20210310154414.3560115-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 1/2] lc-compliance: Add a libcamera compliance tool X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add a compliance tool to ease testing of cameras. In contrast to the unit-tests under test/ that aims to test the internal components of libcamera the compliance tool aims to test application use-cases and to some extend the public API. This change adds the boilerplate code of a simple framework for the creation of tests. The tests aim both to demonstrate the tool and to catch real problems. The tests added are: - Test that if one queues exactly N requests to a camera exactly N requests are eventually completed. - Test that a configured camera can be started and stopped multiple times in an attempt to exercise cleanup code paths otherwise not often tested with 'cam' for example. Signed-off-by: Niklas Söderlund --- * Changes since v1 - Improve language in commit message and comments. - Test all roles as they may exercise different code paths in the pipeline. - Move SimpleCapture to its own .cpp/.h files. * Changes since v2 - Fold in a use-after-free bug fix from Kieran, thanks! --- src/lc-compliance/main.cpp | 139 +++++++++++++++++++++++++ src/lc-compliance/meson.build | 25 +++++ src/lc-compliance/results.cpp | 75 ++++++++++++++ src/lc-compliance/results.h | 45 ++++++++ src/lc-compliance/simple_capture.cpp | 149 +++++++++++++++++++++++++++ src/lc-compliance/simple_capture.h | 54 ++++++++++ src/lc-compliance/single_stream.cpp | 75 ++++++++++++++ src/lc-compliance/tests.h | 16 +++ src/meson.build | 2 + 9 files changed, 580 insertions(+) create mode 100644 src/lc-compliance/main.cpp create mode 100644 src/lc-compliance/meson.build create mode 100644 src/lc-compliance/results.cpp create mode 100644 src/lc-compliance/results.h create mode 100644 src/lc-compliance/simple_capture.cpp create mode 100644 src/lc-compliance/simple_capture.h create mode 100644 src/lc-compliance/single_stream.cpp create mode 100644 src/lc-compliance/tests.h diff --git a/src/lc-compliance/main.cpp b/src/lc-compliance/main.cpp new file mode 100644 index 0000000000000000..e1cbce7eac3df2bc --- /dev/null +++ b/src/lc-compliance/main.cpp @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * main.cpp - lc-compliance - The libcamera compliance tool + */ + +#include +#include +#include + +#include + +#include "../cam/options.h" +#include "tests.h" + +using namespace libcamera; + +class Harness +{ +public: + Harness(); + ~Harness(); + + int exec(int argc, char **argv); + +private: + enum { + OptCamera = 'c', + OptHelp = 'h', + }; + + int parseOptions(int argc, char **argv); + int init(int argc, char **argv); + + OptionsParser::Options options_; + std::unique_ptr cm_; + std::shared_ptr camera_; +}; + +Harness::Harness() +{ + cm_ = std::make_unique(); +} + +Harness::~Harness() +{ + if (camera_) { + camera_->release(); + camera_.reset(); + } + + cm_->stop(); +} + +int Harness::exec(int argc, char **argv) +{ + int ret = init(argc, argv); + if (ret) + return ret; + + std::vector results; + + results.push_back(testSingleStream(camera_)); + + for (const Results &result : results) { + ret = result.summary(); + if (ret) + return ret; + } + + return 0; +} + +int Harness::init(int argc, char **argv) +{ + int ret = parseOptions(argc, argv); + if (ret < 0) + return ret; + + ret = cm_->start(); + if (ret) { + std::cout << "Failed to start camera manager: " + << strerror(-ret) << std::endl; + return ret; + } + + if (!options_.isSet(OptCamera)) { + std::cout << "No camera specified, available cameras:" << std::endl; + for (const std::shared_ptr &cam : cm_->cameras()) + std::cout << "- " << cam.get()->id() << std::endl; + return -ENODEV; + } + + const std::string &cameraId = options_[OptCamera]; + camera_ = cm_->get(cameraId); + if (!camera_) { + std::cout << "Camera " << cameraId << " not found, available cameras:" << std::endl; + for (const std::shared_ptr &cam : cm_->cameras()) + std::cout << "- " << cam.get()->id() << std::endl; + return -ENODEV; + } + + if (camera_->acquire()) { + std::cout << "Failed to acquire camera" << std::endl; + return -EINVAL; + } + + std::cout << "Using camera " << cameraId << std::endl; + + return 0; +} + +int Harness::parseOptions(int argc, char **argv) +{ + OptionsParser parser; + parser.addOption(OptCamera, OptionString, + "Specify which camera to operate on, by id", "camera", + ArgumentRequired, "camera"); + parser.addOption(OptHelp, OptionNone, "Display this help message", + "help"); + + options_ = parser.parse(argc, argv); + if (!options_.valid()) + return -EINVAL; + + if (options_.empty() || options_.isSet(OptHelp)) { + parser.usage(); + return options_.empty() ? -EINVAL : -EINTR; + } + + return 0; +} + +int main(int argc, char **argv) +{ + Harness harness; + return harness.exec(argc, argv) ? EXIT_FAILURE : 0; +} diff --git a/src/lc-compliance/meson.build b/src/lc-compliance/meson.build new file mode 100644 index 0000000000000000..68164537c1055f28 --- /dev/null +++ b/src/lc-compliance/meson.build @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: CC0-1.0 + +libevent = dependency('libevent_pthreads', required : false) + +if not libevent.found() + warning('libevent_pthreads not found, \'lc-compliance\' application will not be compiled') + subdir_done() +endif + +lc_compliance_sources = files([ + '../cam/event_loop.cpp', + '../cam/options.cpp', + 'main.cpp', + 'results.cpp', + 'simple_capture.cpp', + 'single_stream.cpp', +]) + +lc_compliance = executable('lc-compliance', lc_compliance_sources, + dependencies : [ + libatomic, + libcamera_dep, + libevent, + ], + install : true) diff --git a/src/lc-compliance/results.cpp b/src/lc-compliance/results.cpp new file mode 100644 index 0000000000000000..8c42eb2d6822aa60 --- /dev/null +++ b/src/lc-compliance/results.cpp @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * results.cpp - Test result aggregator + */ + +#include "results.h" + +#include + +void Results::add(const Result &result) +{ + if (result.first == Pass) + passed_++; + else if (result.first == Fail) + failed_++; + else if (result.first == Skip) + skipped_++; + + printResult(result); +} + +void Results::add(Status status, const std::string &message) +{ + add({ status, message }); +} + +void Results::fail(const std::string &message) +{ + add(Fail, message); +} + +void Results::pass(const std::string &message) +{ + add(Pass, message); +} + +void Results::skip(const std::string &message) +{ + add(Skip, message); +} + +int Results::summary() const +{ + if (failed_ + passed_ + skipped_ != planned_) { + std::cout << "Planned and executed number of tests differ " + << failed_ + passed_ + skipped_ << " executed " + << planned_ << " planned" << std::endl; + + return -EINVAL; + } + + std::cout << planned_ << " tests executed, " + << passed_ << " tests passed, " + << skipped_ << " tests skipped and " + << failed_ << " tests failed " << std::endl; + + return 0; +} + +void Results::printResult(const Result &result) +{ + std::string prefix; + + /* \todo Make parsable as TAP. */ + if (result.first == Pass) + prefix = "PASS"; + else if (result.first == Fail) + prefix = "FAIL"; + else if (result.first == Skip) + prefix = "SKIP"; + + std::cout << "- " << prefix << " - " << result.second << std::endl; +} diff --git a/src/lc-compliance/results.h b/src/lc-compliance/results.h new file mode 100644 index 0000000000000000..a02fd5ab46edd62c --- /dev/null +++ b/src/lc-compliance/results.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * results.h - Test result aggregator + */ +#ifndef __LC_COMPLIANCE_RESULTS_H__ +#define __LC_COMPLIANCE_RESULTS_H__ + +#include + +class Results +{ +public: + enum Status { + Fail, + Pass, + Skip, + }; + + using Result = std::pair; + + Results(unsigned int planned) + : planned_(planned), passed_(0), failed_(0), skipped_(0) + { + } + + void add(const Result &result); + void add(Status status, const std::string &message); + void fail(const std::string &message); + void pass(const std::string &message); + void skip(const std::string &message); + + int summary() const; + +private: + void printResult(const Result &result); + + unsigned int planned_; + unsigned int passed_; + unsigned int failed_; + unsigned int skipped_; +}; + +#endif /* __LC_COMPLIANCE_RESULTS_H__ */ diff --git a/src/lc-compliance/simple_capture.cpp b/src/lc-compliance/simple_capture.cpp new file mode 100644 index 0000000000000000..fac8db379118efbd --- /dev/null +++ b/src/lc-compliance/simple_capture.cpp @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * simple_capture.cpp - Simple capture helper + */ + +#include "simple_capture.h" + +using namespace libcamera; + +SimpleCapture::SimpleCapture(std::shared_ptr camera) + : camera_(camera), allocator_(std::make_unique(camera)) +{ +} + +SimpleCapture::~SimpleCapture() +{ +} + +Results::Result SimpleCapture::configure(StreamRole role) +{ + config_ = camera_->generateConfiguration({ role }); + + if (config_->validate() != CameraConfiguration::Valid) { + config_.reset(); + return { Results::Fail, "Configuration not valid" }; + } + + if (camera_->configure(config_.get())) { + config_.reset(); + return { Results::Fail, "Failed to configure camera" }; + } + + return { Results::Pass, "Configure camera" }; +} + +Results::Result SimpleCapture::start() +{ + Stream *stream = config_->at(0).stream(); + if (allocator_->allocate(stream) < 0) + return { Results::Fail, "Failed to allocate buffers" }; + + if (camera_->start()) + return { Results::Fail, "Failed to start camera" }; + + camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete); + + return { Results::Pass, "Started camera" }; +} + +Results::Result SimpleCapture::stop() +{ + Stream *stream = config_->at(0).stream(); + + camera_->stop(); + + camera_->requestCompleted.disconnect(this, &SimpleCapture::requestComplete); + + allocator_->free(stream); + + return { Results::Pass, "Stopped camera" }; +} + +/* SimpleCaptureBalanced */ + +SimpleCaptureBalanced::SimpleCaptureBalanced(std::shared_ptr camera) + : SimpleCapture(camera) +{ +} + +Results::Result SimpleCaptureBalanced::capture(unsigned int numRequests) +{ + Results::Result ret = start(); + if (ret.first != Results::Pass) + return ret; + + Stream *stream = config_->at(0).stream(); + const std::vector> &buffers = allocator_->buffers(stream); + + /* No point in testing less requests then the camera depth. */ + if (buffers.size() > numRequests) { + /* Cache buffers.size() before we destroy it in stop() */ + int buffers_size = buffers.size(); + stop(); + + return { Results::Skip, "Camera needs " + std::to_string(buffers_size) + + " requests, can't test only " + std::to_string(numRequests) }; + } + + queueCount_ = 0; + captureCount_ = 0; + captureLimit_ = numRequests; + + /* Queue the recommended number of reqeuests. */ + std::vector> requests; + for (const std::unique_ptr &buffer : buffers) { + std::unique_ptr request = camera_->createRequest(); + if (!request) { + stop(); + return { Results::Fail, "Can't create request" }; + } + + if (request->addBuffer(stream, buffer.get())) { + stop(); + return { Results::Fail, "Can't set buffer for request" }; + } + + if (queueRequest(request.get()) < 0) { + stop(); + return { Results::Fail, "Failed to queue request" }; + } + + requests.push_back(std::move(request)); + } + + /* Run capture session. */ + loop_ = new EventLoop(); + loop_->exec(); + stop(); + delete loop_; + + if (captureCount_ != captureLimit_) + return { Results::Fail, "Got " + std::to_string(captureCount_) + " request, wanted " + std::to_string(captureLimit_) }; + + return { Results::Pass, "Balanced capture of " + std::to_string(numRequests) + " requests" }; +} + +int SimpleCaptureBalanced::queueRequest(Request *request) +{ + queueCount_++; + if (queueCount_ > captureLimit_) + return 0; + + return camera_->queueRequest(request); +} + +void SimpleCaptureBalanced::requestComplete(Request *request) +{ + captureCount_++; + if (captureCount_ >= captureLimit_) { + loop_->exit(0); + return; + } + + request->reuse(Request::ReuseBuffers); + if (queueRequest(request)) + loop_->exit(-EINVAL); +} diff --git a/src/lc-compliance/simple_capture.h b/src/lc-compliance/simple_capture.h new file mode 100644 index 0000000000000000..3a6afc538c623050 --- /dev/null +++ b/src/lc-compliance/simple_capture.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * simple_capture.h - Simple capture helper + */ +#ifndef __LC_COMPLIANCE_SIMPLE_CAPTURE_H__ +#define __LC_COMPLIANCE_SIMPLE_CAPTURE_H__ + +#include + +#include + +#include "../cam/event_loop.h" +#include "results.h" + +class SimpleCapture +{ +public: + Results::Result configure(libcamera::StreamRole role); + +protected: + SimpleCapture(std::shared_ptr camera); + virtual ~SimpleCapture(); + + Results::Result start(); + Results::Result stop(); + + virtual void requestComplete(libcamera::Request *request) = 0; + + EventLoop *loop_; + + std::shared_ptr camera_; + std::unique_ptr allocator_; + std::unique_ptr config_; +}; + +class SimpleCaptureBalanced : public SimpleCapture +{ +public: + SimpleCaptureBalanced(std::shared_ptr camera); + + Results::Result capture(unsigned int numRequests); + +private: + int queueRequest(libcamera::Request *request); + void requestComplete(libcamera::Request *request) override; + + unsigned int queueCount_; + unsigned int captureCount_; + unsigned int captureLimit_; +}; + +#endif /* __LC_COMPLIANCE_SIMPLE_CAPTURE_H__ */ diff --git a/src/lc-compliance/single_stream.cpp b/src/lc-compliance/single_stream.cpp new file mode 100644 index 0000000000000000..0ed6f5dcfb5a516d --- /dev/null +++ b/src/lc-compliance/single_stream.cpp @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * single_stream.cpp - Test a single camera stream + */ + +#include + +#include "simple_capture.h" +#include "tests.h" + +using namespace libcamera; + +Results::Result testRequestBalance(std::shared_ptr camera, + StreamRole role, unsigned int startCycles, + unsigned int numRequests) +{ + SimpleCaptureBalanced capture(camera); + + Results::Result ret = capture.configure(role); + if (ret.first != Results::Pass) + return ret; + + for (unsigned int starts = 0; starts < startCycles; starts++) { + ret = capture.capture(numRequests); + if (ret.first != Results::Pass) + return ret; + } + + return { Results::Pass, "Balanced capture of " + std::to_string(numRequests) + " requests with " + std::to_string(startCycles) + " start cycles" }; +} + +Results testSingleStream(std::shared_ptr camera) +{ + const std::vector> roles = { + { "raw", Raw }, + { "still", StillCapture }, + { "video", VideoRecording }, + { "viewfinder", Viewfinder }, + }; + const std::vector numRequests = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89 }; + + Results results(numRequests.size() * roles.size() * 2); + + if (!camera) + return results; + + for (const auto &role : roles) { + std::cout << "= Test role " << role.first << std::endl; + /* + * Test single capture cycles + * + * Makes sure the camera completes the exact number of requests queued. + * Example failure is a camera that needs N+M requests queued to + * complete N requests to the application. + */ + std::cout << "* Test single capture cycles" << std::endl; + for (unsigned int num : numRequests) + results.add(testRequestBalance(camera, role.second, 1, num)); + + /* + * Test multiple start/stop cycles + * + * Makes sure the camera supports multiple start/stop cycles. + * Example failure is a camera that does not clean up correctly in its + * error path but is only tested by single-capture applications. + */ + std::cout << "* Test multiple start/stop cycles" << std::endl; + for (unsigned int num : numRequests) + results.add(testRequestBalance(camera, role.second, 3, num)); + } + + return results; +} diff --git a/src/lc-compliance/tests.h b/src/lc-compliance/tests.h new file mode 100644 index 0000000000000000..396605214e4b8980 --- /dev/null +++ b/src/lc-compliance/tests.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Google Inc. + * + * tests.h - Test modules + */ +#ifndef __LC_COMPLIANCE_TESTS_H__ +#define __LC_COMPLIANCE_TESTS_H__ + +#include + +#include "results.h" + +Results testSingleStream(std::shared_ptr camera); + +#endif /* __LC_COMPLIANCE_TESTS_H__ */ diff --git a/src/meson.build b/src/meson.build index c908b0675773301d..a8a6a572e4cf9482 100644 --- a/src/meson.build +++ b/src/meson.build @@ -20,6 +20,8 @@ subdir('android') subdir('libcamera') subdir('ipa') +subdir('lc-compliance') + subdir('cam') subdir('qcam')