[{"id":15532,"web_url":"https://patchwork.libcamera.org/comment/15532/","msgid":"<97eb2593-d27b-76e0-0480-cb16e3b556bc@ideasonboard.com>","date":"2021-03-08T17:09:28","subject":"Re: [libcamera-devel] [PATCH v2 1/2] lc-compliance: Add a libcamera\n\tcompliance tool","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"On 08/02/2021 10:21, Niklas Söderlund wrote:\n> Add a compliance tool to ease testing of cameras. In contrast to the\n> unit-tests under test/ that aims to test the internal components of\n> libcamera the compliance tool aims to test application use-cases and to\n> some extend the public API.\n> \n> This change adds the boilerplate code of a simple framework for the\n> creation of tests. The tests aim both to demonstrate the tool and to\n> catch real problems. The tests added are:\n> \n>  - Test that if one queues exactly N requests to a camera exactly N\n>    requests are eventually completed.\n> \n>  - Test that a configured camera can be started and stopped multiple\n>    times in an attempt to exercise cleanup code paths otherwise not\n>    often tested with 'cam' for example.\n> \n> Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>\n> ---\n> * Changes since v1\n> - Improve language in commit message and comments.\n> - Test all roles as they may exercise different code paths in the\n>   pipeline.\n> - Move SimpleCapture to its own .cpp/.h files.\n> ---\n>  src/lc-compliance/main.cpp           | 139 +++++++++++++++++++++++++\n>  src/lc-compliance/meson.build        |  25 +++++\n>  src/lc-compliance/results.cpp        |  75 ++++++++++++++\n>  src/lc-compliance/results.h          |  45 +++++++++\n>  src/lc-compliance/simple_capture.cpp | 145 +++++++++++++++++++++++++++\n>  src/lc-compliance/simple_capture.h   |  54 ++++++++++\n>  src/lc-compliance/single_stream.cpp  |  75 ++++++++++++++\n>  src/lc-compliance/tests.h            |  16 +++\n>  src/meson.build                      |   2 +\n>  9 files changed, 576 insertions(+)\n>  create mode 100644 src/lc-compliance/main.cpp\n>  create mode 100644 src/lc-compliance/meson.build\n>  create mode 100644 src/lc-compliance/results.cpp\n>  create mode 100644 src/lc-compliance/results.h\n>  create mode 100644 src/lc-compliance/simple_capture.cpp\n>  create mode 100644 src/lc-compliance/simple_capture.h\n>  create mode 100644 src/lc-compliance/single_stream.cpp\n>  create mode 100644 src/lc-compliance/tests.h\n> \n> diff --git a/src/lc-compliance/main.cpp b/src/lc-compliance/main.cpp\n> new file mode 100644\n> index 0000000000000000..e1cbce7eac3df2bc\n> --- /dev/null\n> +++ b/src/lc-compliance/main.cpp\n> @@ -0,0 +1,139 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * main.cpp - lc-compliance - The libcamera compliance tool\n> + */\n> +\n> +#include <iomanip>\n> +#include <iostream>\n> +#include <string.h>\n> +\n> +#include <libcamera/libcamera.h>\n> +\n> +#include \"../cam/options.h\"\n> +#include \"tests.h\"\n> +\n> +using namespace libcamera;\n> +\n> +class Harness\n> +{\n> +public:\n> +\tHarness();\n> +\t~Harness();\n> +\n> +\tint exec(int argc, char **argv);\n> +\n> +private:\n> +\tenum {\n> +\t\tOptCamera = 'c',\n> +\t\tOptHelp = 'h',\n> +\t};\n> +\n> +\tint parseOptions(int argc, char **argv);\n> +\tint init(int argc, char **argv);\n> +\n> +\tOptionsParser::Options options_;\n> +\tstd::unique_ptr<CameraManager> cm_;\n> +\tstd::shared_ptr<Camera> camera_;\n> +};\n> +\n> +Harness::Harness()\n> +{\n> +\tcm_ = std::make_unique<CameraManager>();\n> +}\n> +\n> +Harness::~Harness()\n> +{\n> +\tif (camera_) {\n> +\t\tcamera_->release();\n> +\t\tcamera_.reset();\n> +\t}\n> +\n> +\tcm_->stop();\n> +}\n> +\n> +int Harness::exec(int argc, char **argv)\n> +{\n> +\tint ret = init(argc, argv);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\tstd::vector<Results> results;\n> +\n> +\tresults.push_back(testSingleStream(camera_));\n> +\n> +\tfor (const Results &result : results) {\n> +\t\tret = result.summary();\n> +\t\tif (ret)\n> +\t\t\treturn ret;\n> +\t}\n> +\n> +\treturn 0;\n> +}\n> +\n> +int Harness::init(int argc, char **argv)\n> +{\n> +\tint ret = parseOptions(argc, argv);\n> +\tif (ret < 0)\n> +\t\treturn ret;\n> +\n> +\tret = cm_->start();\n> +\tif (ret) {\n> +\t\tstd::cout << \"Failed to start camera manager: \"\n> +\t\t\t  << strerror(-ret) << std::endl;\n> +\t\treturn ret;\n> +\t}\n> +\n> +\tif (!options_.isSet(OptCamera)) {\n> +\t\tstd::cout << \"No camera specified, available cameras:\" << std::endl;\n> +\t\tfor (const std::shared_ptr<Camera> &cam : cm_->cameras())\n> +\t\t\tstd::cout << \"- \" << cam.get()->id() << std::endl;\n> +\t\treturn -ENODEV;\n> +\t}\n> +\n> +\tconst std::string &cameraId = options_[OptCamera];\n> +\tcamera_ = cm_->get(cameraId);\n> +\tif (!camera_) {\n> +\t\tstd::cout << \"Camera \" << cameraId << \" not found, available cameras:\" << std::endl;\n> +\t\tfor (const std::shared_ptr<Camera> &cam : cm_->cameras())\n> +\t\t\tstd::cout << \"- \" << cam.get()->id() << std::endl;\n> +\t\treturn -ENODEV;\n> +\t}\n> +\n> +\tif (camera_->acquire()) {\n> +\t\tstd::cout << \"Failed to acquire camera\" << std::endl;\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tstd::cout << \"Using camera \" << cameraId << std::endl;\n> +\n> +\treturn 0;\n> +}\n> +\n> +int Harness::parseOptions(int argc, char **argv)\n> +{\n> +\tOptionsParser parser;\n> +\tparser.addOption(OptCamera, OptionString,\n> +\t\t\t \"Specify which camera to operate on, by id\", \"camera\",\n> +\t\t\t ArgumentRequired, \"camera\");\n> +\tparser.addOption(OptHelp, OptionNone, \"Display this help message\",\n> +\t\t\t \"help\");\n> +\n> +\toptions_ = parser.parse(argc, argv);\n> +\tif (!options_.valid())\n> +\t\treturn -EINVAL;\n> +\n> +\tif (options_.empty() || options_.isSet(OptHelp)) {\n> +\t\tparser.usage();\n> +\t\treturn options_.empty() ? -EINVAL : -EINTR;\n> +\t}\n> +\n> +\treturn 0;\n> +}\n> +\n> +int main(int argc, char **argv)\n> +{\n> +\tHarness harness;\n> +\treturn harness.exec(argc, argv) ? EXIT_FAILURE : 0;\n> +}\n> diff --git a/src/lc-compliance/meson.build b/src/lc-compliance/meson.build\n> new file mode 100644\n> index 0000000000000000..68164537c1055f28\n> --- /dev/null\n> +++ b/src/lc-compliance/meson.build\n> @@ -0,0 +1,25 @@\n> +# SPDX-License-Identifier: CC0-1.0\n> +\n> +libevent = dependency('libevent_pthreads', required : false)\n> +\n> +if not libevent.found()\n> +    warning('libevent_pthreads not found, \\'lc-compliance\\' application will not be compiled')\n> +    subdir_done()\n> +endif\n> +\n> +lc_compliance_sources = files([\n> +    '../cam/event_loop.cpp',\n> +    '../cam/options.cpp',\n> +    'main.cpp',\n> +    'results.cpp',\n> +    'simple_capture.cpp',\n> +    'single_stream.cpp',\n> +])\n> +\n> +lc_compliance  = executable('lc-compliance', lc_compliance_sources,\n> +                  dependencies : [\n> +                      libatomic,\n> +                      libcamera_dep,\n> +                      libevent,\n> +                  ],\n> +                  install : true)\n> diff --git a/src/lc-compliance/results.cpp b/src/lc-compliance/results.cpp\n> new file mode 100644\n> index 0000000000000000..8c42eb2d6822aa60\n> --- /dev/null\n> +++ b/src/lc-compliance/results.cpp\n> @@ -0,0 +1,75 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * results.cpp - Test result aggregator\n> + */\n> +\n> +#include \"results.h\"\n> +\n> +#include <iostream>\n> +\n> +void Results::add(const Result &result)\n> +{\n> +\tif (result.first == Pass)\n> +\t\tpassed_++;\n> +\telse if (result.first == Fail)\n> +\t\tfailed_++;\n> +\telse if (result.first == Skip)\n> +\t\tskipped_++;\n> +\n> +\tprintResult(result);\n> +}\n> +\n> +void Results::add(Status status, const std::string &message)\n> +{\n> +\tadd({ status, message });\n> +}\n> +\n> +void Results::fail(const std::string &message)\n> +{\n> +\tadd(Fail, message);\n> +}\n> +\n> +void Results::pass(const std::string &message)\n> +{\n> +\tadd(Pass, message);\n> +}\n> +\n> +void Results::skip(const std::string &message)\n> +{\n> +\tadd(Skip, message);\n> +}\n> +\n> +int Results::summary() const\n> +{\n> +\tif (failed_ + passed_ + skipped_ != planned_) {\n> +\t\tstd::cout << \"Planned and executed number of tests differ \"\n> +\t\t\t  << failed_ + passed_ + skipped_ << \" executed \"\n> +\t\t\t  << planned_ << \" planned\" << std::endl;\n> +\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tstd::cout << planned_ << \" tests executed, \"\n> +\t\t  << passed_ << \" tests passed, \"\n> +\t\t  << skipped_ << \" tests skipped and \"\n> +\t\t  << failed_ << \" tests failed \" << std::endl;\n> +\n> +\treturn 0;\n> +}\n> +\n> +void Results::printResult(const Result &result)\n> +{\n> +\tstd::string prefix;\n> +\n> +\t/* \\todo Make parsable as TAP. */\n> +\tif (result.first == Pass)\n> +\t\tprefix = \"PASS\";\n> +\telse if (result.first == Fail)\n> +\t\tprefix = \"FAIL\";\n> +\telse if (result.first == Skip)\n> +\t\tprefix = \"SKIP\";\n> +\n> +\tstd::cout << \"- \" << prefix << \" - \" << result.second << std::endl;\n> +}\n> diff --git a/src/lc-compliance/results.h b/src/lc-compliance/results.h\n> new file mode 100644\n> index 0000000000000000..a02fd5ab46edd62c\n> --- /dev/null\n> +++ b/src/lc-compliance/results.h\n> @@ -0,0 +1,45 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * results.h - Test result aggregator\n> + */\n> +#ifndef __LC_COMPLIANCE_RESULTS_H__\n> +#define __LC_COMPLIANCE_RESULTS_H__\n> +\n> +#include <string>\n> +\n> +class Results\n> +{\n> +public:\n> +\tenum Status {\n> +\t\tFail,\n> +\t\tPass,\n> +\t\tSkip,\n> +\t};\n> +\n> +\tusing Result = std::pair<Status, std::string>;\n> +\n> +\tResults(unsigned int planned)\n> +\t\t: planned_(planned), passed_(0), failed_(0), skipped_(0)\n> +\t{\n> +\t}\n> +\n> +\tvoid add(const Result &result);\n> +\tvoid add(Status status, const std::string &message);\n> +\tvoid fail(const std::string &message);\n> +\tvoid pass(const std::string &message);\n> +\tvoid skip(const std::string &message);\n> +\n> +\tint summary() const;\n> +\n> +private:\n> +\tvoid printResult(const Result &result);\n> +\n> +\tunsigned int planned_;\n> +\tunsigned int passed_;\n> +\tunsigned int failed_;\n> +\tunsigned int skipped_;\n> +};\n> +\n> +#endif /* __LC_COMPLIANCE_RESULTS_H__ */\n> diff --git a/src/lc-compliance/simple_capture.cpp b/src/lc-compliance/simple_capture.cpp\n> new file mode 100644\n> index 0000000000000000..e6699b689ace180d\n> --- /dev/null\n> +++ b/src/lc-compliance/simple_capture.cpp\n> @@ -0,0 +1,145 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * simple_capture.cpp - Simple capture helper\n> + */\n> +\n> +#include \"simple_capture.h\"\n> +\n> +using namespace libcamera;\n> +\n> +SimpleCapture::SimpleCapture(std::shared_ptr<Camera> camera)\n> +\t: camera_(camera), allocator_(std::make_unique<FrameBufferAllocator>(camera))\n> +{\n> +}\n> +\n> +SimpleCapture::~SimpleCapture()\n> +{\n> +}\n> +\n> +Results::Result SimpleCapture::configure(StreamRole role)\n> +{\n> +\tconfig_ = camera_->generateConfiguration({ role });\n> +\n> +\tif (config_->validate() != CameraConfiguration::Valid) {\n> +\t\tconfig_.reset();\n> +\t\treturn { Results::Fail, \"Configuration not valid\" };\n> +\t}\n> +\n> +\tif (camera_->configure(config_.get())) {\n> +\t\tconfig_.reset();\n> +\t\treturn { Results::Fail, \"Failed to configure camera\" };\n> +\t}\n> +\n> +\treturn { Results::Pass, \"Configure camera\" };\n> +}\n> +\n> +Results::Result SimpleCapture::start()\n> +{\n> +\tStream *stream = config_->at(0).stream();\n> +\tif (allocator_->allocate(stream) < 0)\n> +\t\treturn { Results::Fail, \"Failed to allocate buffers\" };\n> +\n> +\tif (camera_->start())\n> +\t\treturn { Results::Fail, \"Failed to start camera\" };\n> +\n> +\tcamera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> +\n> +\treturn { Results::Pass, \"Started camera\" };\n> +}\n> +\n> +Results::Result SimpleCapture::stop()\n> +{\n> +\tStream *stream = config_->at(0).stream();\n> +\n> +\tcamera_->stop();\n> +\n> +\tcamera_->requestCompleted.disconnect(this, &SimpleCapture::requestComplete);\n> +\n> +\tallocator_->free(stream);\n> +\n> +\treturn { Results::Pass, \"Stopped camera\" };\n> +}\n> +\n> +/* SimpleCaptureBalanced */\n> +\n> +SimpleCaptureBalanced::SimpleCaptureBalanced(std::shared_ptr<Camera> camera)\n> +\t: SimpleCapture(camera)\n> +{\n> +}\n> +\n> +Results::Result SimpleCaptureBalanced::capture(unsigned int numRequests)\n> +{\n> +\tResults::Result ret = start();\n> +\tif (ret.first != Results::Pass)\n> +\t\treturn ret;\n> +\n> +\tStream *stream = config_->at(0).stream();\n> +\tconst std::vector<std::unique_ptr<FrameBuffer>> &buffers = allocator_->buffers(stream);\n> +\n> +\t/* No point in testing less requests then the camera depth. */\n> +\tif (buffers.size() > numRequests) {\n> +\t\tstop();\n> +\t\treturn { Results::Skip, \"Camera needs \" + std::to_string(buffers.size()) + \" requests, can't test only \" + std::to_string(numRequests) };\n\nThis accesses buffers (buffers.size()) after it's been destroyed in\nstop();, and flags a use-after-free condition in valgrind.\n\nFixup patch incoming.\n\n> +\t}\n> +\n> +\tqueueCount_ = 0;\n> +\tcaptureCount_ = 0;\n> +\tcaptureLimit_ = numRequests;\n> +\n> +\t/* Queue the recommended number of reqeuests. */\n> +\tstd::vector<std::unique_ptr<libcamera::Request>> requests;\n> +\tfor (const std::unique_ptr<FrameBuffer> &buffer : buffers) {\n> +\t\tstd::unique_ptr<Request> request = camera_->createRequest();\n> +\t\tif (!request) {\n> +\t\t\tstop();\n> +\t\t\treturn { Results::Fail, \"Can't create request\" };\n> +\t\t}\n> +\n> +\t\tif (request->addBuffer(stream, buffer.get())) {\n> +\t\t\tstop();\n> +\t\t\treturn { Results::Fail, \"Can't set buffer for request\" };\n> +\t\t}\n> +\n> +\t\tif (queueRequest(request.get()) < 0) {\n> +\t\t\tstop();\n> +\t\t\treturn { Results::Fail, \"Failed to queue request\" };\n> +\t\t}\n> +\n> +\t\trequests.push_back(std::move(request));\n> +\t}\n> +\n> +\t/* Run capture session. */\n> +\tloop_ = new EventLoop();\n> +\tloop_->exec();\n> +\tstop();\n> +\tdelete loop_;\n> +\n> +\tif (captureCount_ != captureLimit_)\n> +\t\treturn { Results::Fail, \"Got \" + std::to_string(captureCount_) + \" request, wanted \" + std::to_string(captureLimit_) };\n> +\n> +\treturn { Results::Pass, \"Balanced capture of \" + std::to_string(numRequests) + \" requests\" };\n> +}\n> +\n> +int SimpleCaptureBalanced::queueRequest(Request *request)\n> +{\n> +\tqueueCount_++;\n> +\tif (queueCount_ > captureLimit_)\n> +\t\treturn 0;\n> +\n> +\treturn camera_->queueRequest(request);\n> +}\n> +\n> +void SimpleCaptureBalanced::requestComplete(Request *request)\n> +{\n> +\tcaptureCount_++;\n> +\tif (captureCount_ >= captureLimit_) {\n> +\t\tloop_->exit(0);\n> +\t\treturn;\n> +\t}\n> +\n> +\trequest->reuse(Request::ReuseBuffers);\n> +\tif (queueRequest(request))\n> +\t\tloop_->exit(-EINVAL);\n> +}\n> diff --git a/src/lc-compliance/simple_capture.h b/src/lc-compliance/simple_capture.h\n> new file mode 100644\n> index 0000000000000000..3a6afc538c623050\n> --- /dev/null\n> +++ b/src/lc-compliance/simple_capture.h\n> @@ -0,0 +1,54 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * simple_capture.h - Simple capture helper\n> + */\n> +#ifndef __LC_COMPLIANCE_SIMPLE_CAPTURE_H__\n> +#define __LC_COMPLIANCE_SIMPLE_CAPTURE_H__\n> +\n> +#include <memory>\n> +\n> +#include <libcamera/libcamera.h>\n> +\n> +#include \"../cam/event_loop.h\"\n> +#include \"results.h\"\n> +\n> +class SimpleCapture\n> +{\n> +public:\n> +\tResults::Result configure(libcamera::StreamRole role);\n> +\n> +protected:\n> +\tSimpleCapture(std::shared_ptr<libcamera::Camera> camera);\n> +\tvirtual ~SimpleCapture();\n> +\n> +\tResults::Result start();\n> +\tResults::Result stop();\n> +\n> +\tvirtual void requestComplete(libcamera::Request *request) = 0;\n> +\n> +\tEventLoop *loop_;\n> +\n> +\tstd::shared_ptr<libcamera::Camera> camera_;\n> +\tstd::unique_ptr<libcamera::FrameBufferAllocator> allocator_;\n> +\tstd::unique_ptr<libcamera::CameraConfiguration> config_;\n> +};\n> +\n> +class SimpleCaptureBalanced : public SimpleCapture\n> +{\n> +public:\n> +\tSimpleCaptureBalanced(std::shared_ptr<libcamera::Camera> camera);\n> +\n> +\tResults::Result capture(unsigned int numRequests);\n> +\n> +private:\n> +\tint queueRequest(libcamera::Request *request);\n> +\tvoid requestComplete(libcamera::Request *request) override;\n> +\n> +\tunsigned int queueCount_;\n> +\tunsigned int captureCount_;\n> +\tunsigned int captureLimit_;\n> +};\n> +\n> +#endif /* __LC_COMPLIANCE_SIMPLE_CAPTURE_H__ */\n> diff --git a/src/lc-compliance/single_stream.cpp b/src/lc-compliance/single_stream.cpp\n> new file mode 100644\n> index 0000000000000000..0ed6f5dcfb5a516d\n> --- /dev/null\n> +++ b/src/lc-compliance/single_stream.cpp\n> @@ -0,0 +1,75 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * single_stream.cpp - Test a single camera stream\n> + */\n> +\n> +#include <iostream>\n> +\n> +#include \"simple_capture.h\"\n> +#include \"tests.h\"\n> +\n> +using namespace libcamera;\n> +\n> +Results::Result testRequestBalance(std::shared_ptr<Camera> camera,\n> +\t\t\t\t   StreamRole role, unsigned int startCycles,\n> +\t\t\t\t   unsigned int numRequests)\n> +{\n> +\tSimpleCaptureBalanced capture(camera);\n> +\n> +\tResults::Result ret = capture.configure(role);\n> +\tif (ret.first != Results::Pass)\n> +\t\treturn ret;\n> +\n> +\tfor (unsigned int starts = 0; starts < startCycles; starts++) {\n> +\t\tret = capture.capture(numRequests);\n> +\t\tif (ret.first != Results::Pass)\n> +\t\t\treturn ret;\n> +\t}\n> +\n> +\treturn { Results::Pass, \"Balanced capture of \" + std::to_string(numRequests) + \" requests with \" + std::to_string(startCycles) + \" start cycles\" };\n> +}\n> +\n> +Results testSingleStream(std::shared_ptr<Camera> camera)\n> +{\n> +\tconst std::vector<std::pair<std::string, StreamRole>> roles = {\n> +\t\t{ \"raw\", Raw },\n> +\t\t{ \"still\", StillCapture },\n> +\t\t{ \"video\", VideoRecording },\n> +\t\t{ \"viewfinder\", Viewfinder },\n> +\t};\n> +\tconst std::vector<unsigned int> numRequests = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89 };\n> +\n> +\tResults results(numRequests.size() * roles.size() * 2);\n> +\n> +\tif (!camera)\n> +\t\treturn results;\n> +\n> +\tfor (const auto &role : roles) {\n> +\t\tstd::cout << \"= Test role \" << role.first << std::endl;\n> +\t\t/*\n> +\t\t * Test single capture cycles\n> +\t\t *\n> +\t\t * Makes sure the camera completes the exact number of requests queued.\n> +\t\t * Example failure is a camera that needs N+M requests queued to\n> +\t\t * complete N requests to the application.\n> +\t\t */\n> +\t\tstd::cout << \"* Test single capture cycles\" << std::endl;\n> +\t\tfor (unsigned int num : numRequests)\n> +\t\t\tresults.add(testRequestBalance(camera, role.second, 1, num));\n> +\n> +\t\t/*\n> +\t\t * Test multiple start/stop cycles\n> +\t\t *\n> +\t\t * Makes sure the camera supports multiple start/stop cycles.\n> +\t\t * Example failure is a camera that does not clean up correctly in its\n> +\t\t * error path but is only tested by single-capture applications.\n> +\t\t */\n> +\t\tstd::cout << \"* Test multiple start/stop cycles\" << std::endl;\n> +\t\tfor (unsigned int num : numRequests)\n> +\t\t\tresults.add(testRequestBalance(camera, role.second, 3, num));\n> +\t}\n> +\n> +\treturn results;\n> +}\n> diff --git a/src/lc-compliance/tests.h b/src/lc-compliance/tests.h\n> new file mode 100644\n> index 0000000000000000..396605214e4b8980\n> --- /dev/null\n> +++ b/src/lc-compliance/tests.h\n> @@ -0,0 +1,16 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Google Inc.\n> + *\n> + * tests.h - Test modules\n> + */\n> +#ifndef __LC_COMPLIANCE_TESTS_H__\n> +#define __LC_COMPLIANCE_TESTS_H__\n> +\n> +#include <libcamera/libcamera.h>\n> +\n> +#include \"results.h\"\n> +\n> +Results testSingleStream(std::shared_ptr<libcamera::Camera> camera);\n> +\n> +#endif /* __LC_COMPLIANCE_TESTS_H__ */\n> diff --git a/src/meson.build b/src/meson.build\n> index 4b75f05878bcb702..a8e1af7adf2ca9c8 100644\n> --- a/src/meson.build\n> +++ b/src/meson.build\n> @@ -18,6 +18,8 @@ subdir('android')\n>  subdir('libcamera')\n>  subdir('ipa')\n>  \n> +subdir('lc-compliance')\n> +\n>  subdir('cam')\n>  subdir('qcam')\n>  \n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 5B8EABD80C\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon,  8 Mar 2021 17:09:34 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id CE65268AAE;\n\tMon,  8 Mar 2021 18:09:33 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CC36C68A9F\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon,  8 Mar 2021 18:09:31 +0100 (CET)","from [192.168.0.20]\n\t(cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 5108CE7B;\n\tMon,  8 Mar 2021 18:09:31 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"BYbxfUvE\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1615223371;\n\tbh=qW14WfGTTCfHUdJC1p4HbHUnmsojcBrxI+AJ2LZinKI=;\n\th=Reply-To:Subject:To:References:From:Date:In-Reply-To:From;\n\tb=BYbxfUvE363oDpqlF87bHuQgxG4UsJnJJfp1V/Nfu3XfCHPw2SETVugID7Ws+6YdV\n\t0kyS0wwq8VWLQqipR7YTwOG+bqaelGQyGiIclVrVpTZoZIK51DCExf94xL70q16y/C\n\twZpRayK67MgLWkN33zDM8I3QEDBevTTdL/TRV1Zw=","To":"=?utf-8?q?Niklas_S=C3=B6derlund?= <niklas.soderlund@ragnatech.se>,\n\tlibcamera-devel@lists.libcamera.org","References":"<20210208102137.2164282-1-niklas.soderlund@ragnatech.se>\n\t<20210208102137.2164282-2-niklas.soderlund@ragnatech.se>","From":"Kieran Bingham <kieran.bingham@ideasonboard.com>","Autocrypt":"addr=kieran.bingham@ideasonboard.com; keydata=\n\tmQINBFYE/WYBEACs1PwjMD9rgCu1hlIiUA1AXR4rv2v+BCLUq//vrX5S5bjzxKAryRf0uHat\n\tV/zwz6hiDrZuHUACDB7X8OaQcwhLaVlq6byfoBr25+hbZG7G3+5EUl9cQ7dQEdvNj6V6y/SC\n\trRanWfelwQThCHckbobWiQJfK9n7rYNcPMq9B8e9F020LFH7Kj6YmO95ewJGgLm+idg1Kb3C\n\tpotzWkXc1xmPzcQ1fvQMOfMwdS+4SNw4rY9f07Xb2K99rjMwZVDgESKIzhsDB5GY465sCsiQ\n\tcSAZRxqE49RTBq2+EQsbrQpIc8XiffAB8qexh5/QPzCmR4kJgCGeHIXBtgRj+nIkCJPZvZtf\n\tKr2EAbc6tgg6DkAEHJb+1okosV09+0+TXywYvtEop/WUOWQ+zo+Y/OBd+8Ptgt1pDRyOBzL8\n\tRXa8ZqRf0Mwg75D+dKntZeJHzPRJyrlfQokngAAs4PaFt6UfS+ypMAF37T6CeDArQC41V3ko\n\tlPn1yMsVD0p+6i3DPvA/GPIksDC4owjnzVX9kM8Zc5Cx+XoAN0w5Eqo4t6qEVbuettxx55gq\n\t8K8FieAjgjMSxngo/HST8TpFeqI5nVeq0/lqtBRQKumuIqDg+Bkr4L1V/PSB6XgQcOdhtd36\n\tOe9X9dXB8YSNt7VjOcO7BTmFn/Z8r92mSAfHXpb07YJWJosQOQARAQABtDBLaWVyYW4gQmlu\n\tZ2hhbSA8a2llcmFuLmJpbmdoYW1AaWRlYXNvbmJvYXJkLmNvbT6JAlcEEwEKAEECGwMFCwkI\n\tBwIGFQgJCgsCBBYCAwECHgECF4ACGQEWIQSQLdeYP70o/eNy1HqhHkZyEKRh/QUCXWTtygUJ\n\tCyJXZAAKCRChHkZyEKRh/f8dEACTDsbLN2nioNZMwyLuQRUAFcXNolDX48xcUXsWS2QjxaPm\n\tVsJx8Uy8aYkS85mdPBh0C83OovQR/OVbr8AxhGvYqBs3nQvbWuTl/+4od7DfK2VZOoKBAu5S\n\tQK2FYuUcikDqYcFWJ8DQnubxfE8dvzojHEkXw0sA4igINHDDFX3HJGZtLio+WpEFQtCbfTAG\n\tYZslasz1YZRbwEdSsmO3/kqy5eMnczlm8a21A3fKUo3g8oAZEFM+f4DUNzqIltg31OAB/kZS\n\tenKZQ/SWC8PmLg/ZXBrReYakxXtkP6w3FwMlzOlhGxqhIRNiAJfXJBaRhuUWzPOpEDE9q5YJ\n\tBmqQL2WJm1VSNNVxbXJHpaWMH1sA2R00vmvRrPXGwyIO0IPYeUYQa3gsy6k+En/aMQJd27dp\n\taScf9am9PFICPY5T4ppneeJLif2lyLojo0mcHOV+uyrds9XkLpp14GfTkeKPdPMrLLTsHRfH\n\tfA4I4OBpRrEPiGIZB/0im98MkGY/Mu6qxeZmYLCcgD6qz4idOvfgVOrNh+aA8HzIVR+RMW8H\n\tQGBN9f0E3kfwxuhl3omo6V7lDw8XOdmuWZNC9zPq1UfryVHANYbLGz9KJ4Aw6M+OgBC2JpkD\n\thXMdHUkC+d20dwXrwHTlrJi1YNp6rBc+xald3wsUPOZ5z8moTHUX/uPA/qhGsbkCDQRWBP1m\n\tARAAzijkb+Sau4hAncr1JjOY+KyFEdUNxRy+hqTJdJfaYihxyaj0Ee0P0zEi35CbE6lgU0Uz\n\ttih9fiUbSV3wfsWqg1Ut3/5rTKu7kLFp15kF7eqvV4uezXRD3Qu4yjv/rMmEJbbD4cTvGCYI\n\td6MDC417f7vK3hCbCVIZSp3GXxyC1LU+UQr3fFcOyCwmP9vDUR9JV0BSqHHxRDdpUXE26Dk6\n\tmhf0V1YkspE5St814ETXpEus2urZE5yJIUROlWPIL+hm3NEWfAP06vsQUyLvr/GtbOT79vXl\n\tEn1aulcYyu20dRRxhkQ6iILaURcxIAVJJKPi8dsoMnS8pB0QW12AHWuirPF0g6DiuUfPmrA5\n\tPKe56IGlpkjc8cO51lIxHkWTpCMWigRdPDexKX+Sb+W9QWK/0JjIc4t3KBaiG8O4yRX8ml2R\n\t+rxfAVKM6V769P/hWoRGdgUMgYHFpHGSgEt80OKK5HeUPy2cngDUXzwrqiM5Sz6Od0qw5pCk\n\tNlXqI0W/who0iSVM+8+RmyY0OEkxEcci7rRLsGnM15B5PjLJjh1f2ULYkv8s4SnDwMZ/kE04\n\t/UqCMK/KnX8pwXEMCjz0h6qWNpGwJ0/tYIgQJZh6bqkvBrDogAvuhf60Sogw+mH8b+PBlx1L\n\toeTK396wc+4c3BfiC6pNtUS5GpsPMMjYMk7kVvEAEQEAAYkCPAQYAQoAJgIbDBYhBJAt15g/\n\tvSj943LUeqEeRnIQpGH9BQJdizzIBQkLSKZiAAoJEKEeRnIQpGH9eYgQAJpjaWNgqNOnMTmD\n\tMJggbwjIotypzIXfhHNCeTkG7+qCDlSaBPclcPGYrTwCt0YWPU2TgGgJrVhYT20ierN8LUvj\n\t6qOPTd+Uk7NFzL65qkh80ZKNBFddx1AabQpSVQKbdcLb8OFs85kuSvFdgqZwgxA1vl4TFhNz\n\tPZ79NAmXLackAx3sOVFhk4WQaKRshCB7cSl+RIng5S/ThOBlwNlcKG7j7W2MC06BlTbdEkUp\n\tECzuuRBv8wX4OQl+hbWbB/VKIx5HKlLu1eypen/5lNVzSqMMIYkkZcjV2SWQyUGxSwq0O/sx\n\tS0A8/atCHUXOboUsn54qdxrVDaK+6jIAuo8JiRWctP16KjzUM7MO0/+4zllM8EY57rXrj48j\n\tsbEYX0YQnzaj+jO6kJtoZsIaYR7rMMq9aUAjyiaEZpmP1qF/2sYenDx0Fg2BSlLvLvXM0vU8\n\tpQk3kgDu7kb/7PRYrZvBsr21EIQoIjXbZxDz/o7z95frkP71EaICttZ6k9q5oxxA5WC6sTXc\n\tMW8zs8avFNuA9VpXt0YupJd2ijtZy2mpZNG02fFVXhIn4G807G7+9mhuC4XG5rKlBBUXTvPU\n\tAfYnB4JBDLmLzBFavQfvonSfbitgXwCG3vS+9HEwAjU30Bar1PEOmIbiAoMzuKeRm2LVpmq4\n\tWZw01QYHU/GUV/zHJSFk","Organization":"Ideas on Board","Message-ID":"<97eb2593-d27b-76e0-0480-cb16e3b556bc@ideasonboard.com>","Date":"Mon, 8 Mar 2021 17:09:28 +0000","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101\n\tThunderbird/68.10.0","MIME-Version":"1.0","In-Reply-To":"<20210208102137.2164282-2-niklas.soderlund@ragnatech.se>","Content-Language":"en-GB","Subject":"Re: [libcamera-devel] [PATCH v2 1/2] lc-compliance: Add a libcamera\n\tcompliance tool","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","Reply-To":"kieran.bingham@ideasonboard.com","Content-Type":"text/plain; charset=\"utf-8\"","Content-Transfer-Encoding":"base64","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]