[{"id":28805,"web_url":"https://patchwork.libcamera.org/comment/28805/","msgid":"<9810ad7e-02a7-4e86-848b-fdeffe610ec3@ideasonboard.com>","date":"2024-02-29T17:08:52","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Just to clarify. This patch fails to run on master. I wanted to collect\nsome feedback on the overall idea and if my assumptions in the tests are\ncorrect.\n\nCheers,\nStefan\n\nAm 29.02.24 um 18:01 schrieb Stefan Klug:\n> These tests check if controls (only exposure time and analogue gain at\n> the moment) get applied on the frame they were requested for.\n> \n> This is tested by looking at the metadata and additionally by\n> calculating a mean brightness on a centered rect of 20x20 pixels.\n> \n> Until today, these tests where only run on a project specific branch\n> with a modified simple pipeline. In theory they should pass on a\n> current master :-)\n> \n> Current test setup: imx219 with simple pipeline on an imx8mp.\n> Modifications of either the exposure delay or the gain delay in\n> the camera_sensor class resulted in test failures.\n> Which is exactly what this test shall proove.\n> \n> Signed-off-by: Stefan Klug \n> ---\n> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> src/apps/lc-compliance/meson.build | 2 +\n> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> src/apps/lc-compliance/simple_capture.h | 2 +-\n> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> 8 files changed, 487 insertions(+), 3 deletions(-)\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> create mode 100644 src/apps/lc-compliance/time_sheet.h\n> \n> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> index 1dcfcf92..43fe59f3 100644\n> --- a/src/apps/lc-compliance/capture_test.cpp\n> +++ b/src/apps/lc-compliance/capture_test.cpp\n> @@ -11,6 +11,7 @@\n> #include \n> \n> #include \"environment.h\"\n> +#include \"per_frame_controls.h\"\n> #include \"simple_capture.h\"\n> \n> using namespace libcamera;\n> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> \t\t\t testing::Combine(testing::ValuesIn(ROLES),\n> \t\t\t\t\t testing::ValuesIn(NUMREQUESTS)),\n> \t\t\t SingleStream::nameParameters);\n> +\n> +/*\n> + * Test Per frame controls\n> + */\n> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> +{\n> +\tPerFrameControls capture(camera_);\n> +\tcapture.configure(StreamRole::Viewfinder);\n> +\tcapture.testFramePreciseExposureChange();\n> +}\n> +\n> +TEST_F(SingleStream, testFramePreciseGainChange)\n> +{\n> +\tPerFrameControls capture(camera_);\n> +\tcapture.configure(StreamRole::Viewfinder);\n> +\tcapture.testFramePreciseGainChange();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> +{\n> +\tPerFrameControls capture(camera_);\n> +\tcapture.configure(StreamRole::Viewfinder);\n> +\tcapture.testExposureGainIsAppliedOnFirstFrame();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> +{\n> +\tPerFrameControls capture(camera_);\n> +\tcapture.configure(StreamRole::Viewfinder);\n> +\tcapture.testExposureGainFromFirstRequestGetsApplied();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> +{\n> +\tPerFrameControls capture(camera_);\n> +\tcapture.configure(StreamRole::Viewfinder);\n> +\tcapture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +}\n> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> index c792f072..2a6f52af 100644\n> --- a/src/apps/lc-compliance/meson.build\n> +++ b/src/apps/lc-compliance/meson.build\n> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> 'capture_test.cpp',\n> 'environment.cpp',\n> 'main.cpp',\n> + 'per_frame_controls.cpp',\n> 'simple_capture.cpp',\n> + 'time_sheet.cpp',\n> ])\n> \n> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> new file mode 100644\n> index 00000000..70fc44ac\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> @@ -0,0 +1,214 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.cpp - Tests for per frame controls\n> + */\n> +#include \"per_frame_controls.h\"\n> +\n> +#include \n> +\n> +#include \"time_sheet.h\"\n> +\n> +using namespace libcamera;\n> +\n> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> +\t: SimpleCapture(camera)\n> +{\n> +}\n> +\n> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> +{\n> +\tControlList ctrls(camera_->controls().idmap());\n> +\t/* Ensure defined default values */\n> +\tctrls.set(controls::AeEnable, false);\n> +\tctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> +\tctrls.set(controls::ExposureTime, 10000);\n> +\tctrls.set(controls::AnalogueGain, 1.0);\n> +\n> +\tif (controls) {\n> +\t\tctrls.merge(*controls, true);\n> +\t}\n> +\n> +\tstart(&ctrls);\n> +\n> +\tqueueCount_ = 0;\n> +\tcaptureCount_ = 0;\n> +\tcaptureLimit_ = framesToCapture;\n> +\n> +\tauto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> +\ttimeSheet_ = timeSheet;\n> +\treturn timeSheet;\n> +}\n> +\n> +int PerFrameControls::queueRequest(Request *request)\n> +{\n> +\tqueueCount_++;\n> +\tif (queueCount_ > captureLimit_)\n> +\t\treturn 0;\n> +\n> +\tauto ts = timeSheet_.lock();\n> +\tif (ts) {\n> +\t\tts->prepareForQueue(request, queueCount_ - 1);\n> +\t}\n> +\n> +\treturn camera_->queueRequest(request);\n> +}\n> +\n> +void PerFrameControls::requestComplete(Request *request)\n> +{\n> +\tauto ts = timeSheet_.lock();\n> +\tif (ts) {\n> +\t\tts->handleCompleteRequest(request);\n> +\t}\n> +\n> +\tcaptureCount_++;\n> +\tif (captureCount_ >= captureLimit_) {\n> +\t\tloop_->exit(0);\n> +\t\treturn;\n> +\t}\n> +\n> +\trequest->reuse(Request::ReuseBuffers);\n> +\tif (queueRequest(request))\n> +\t\tloop_->exit(-EINVAL);\n> +}\n> +\n> +void PerFrameControls::runCaptureSession()\n> +{\n> +\tStream *stream = config_->at(0).stream();\n> +\tconst std::vector> &buffers = allocator_->buffers(stream);\n> +\n> +\t/* Queue the recommended number of reqeuests. */\n> +\tfor (const std::unique_ptr &buffer : buffers) {\n> +\t\tstd::unique_ptr request = camera_->createRequest();\n> +\t\trequest->addBuffer(stream, buffer.get());\n> +\t\tqueueRequest(request.get());\n> +\t\trequests_.push_back(std::move(request));\n> +\t}\n> +\n> +\t/* Run capture session. */\n> +\tloop_ = new EventLoop();\n> +\tloop_->exec();\n> +\tstop();\n> +\tdelete loop_;\n> +}\n> +\n> +void PerFrameControls::testFramePreciseExposureChange()\n> +{\n> +\tauto timeSheet = startCaptureWithTimeSheet(10);\n> +\tauto &ts = *timeSheet;\n> +\n> +\n> +\tts[3].controls().set(controls::ExposureTime, 5000);\n> +\t//wait a few frames to settle\n> +\tts[6].controls().set(controls::ExposureTime, 20000);\n> +\tts.printAllInfos();\n> +\n> +\trunCaptureSession();\n> +\n> +\tEXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> +\tEXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> +\n> +\t/* No increase just before setting exposure */\n> +\tEXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> +\t/*\n> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> + * This should be investigated.\n> + */\n> +\tEXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> +\t\t\t\t\t\t << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> +\t/* No increase just after setting exposure */\n> +\tEXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +\n> +\t/* No increase just after setting exposure */\n> +\tEXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> +}\n> +\n> +void PerFrameControls::testFramePreciseGainChange()\n> +{\n> +\tauto timeSheet = startCaptureWithTimeSheet(10);\n> +\tauto &ts = *timeSheet;\n> +\n> +\tts[3].controls().set(controls::AnalogueGain, 1.0);\n> +\t//wait a few frames to settle\n> +\tts[6].controls().set(controls::AnalogueGain, 4.0);\n> +\n> +\trunCaptureSession();\n> +\n> +\tEXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> +\tEXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +\n> +\t/* No increase just before setting gain */\n> +\tEXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> +\t/*\n> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> + */\n> +\tEXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> +\t\t\t\t\t\t << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> +\t/* No increase just after setting gain */\n> +\tEXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> +{\n> +\tauto timeSheet = startCaptureWithTimeSheet(5);\n> +\tauto &ts = *timeSheet;\n> +\n> +\tts[0].controls().set(controls::ExposureTime, 10000);\n> +\tts[0].controls().set(controls::AnalogueGain, 4.0);\n> +\n> +\trunCaptureSession();\n> +\n> +\t/* We expect it to be applied after 3 frames, the latest*/\n> +\tEXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> +\tEXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> +{\n> +\tauto timeSheet = startCaptureWithTimeSheet(5);\n> +\tauto &ts = *timeSheet;\n> +\n> +\tts[0].controls().set(controls::ExposureTime, 8000);\n> +\tts[0].controls().set(controls::AnalogueGain, 2.0);\n> +\tts[1].controls().set(controls::ExposureTime, 10000);\n> +\tts[1].controls().set(controls::AnalogueGain, 4.0);\n> +\n> +\trunCaptureSession();\n> +\n> +\t/* We expect it to be applied after 3 frames, the latest*/\n> +\tEXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> +\tEXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> +{\n> +\tControlList startValues;\n> +\tstartValues.set(controls::ExposureTime, 5000);\n> +\tstartValues.set(controls::AnalogueGain, 1.0);\n> +\n> +\tauto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> +\trunCaptureSession();\n> +\n> +\tEXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> +\tEXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> +\n> +\t/* Second capture with different values to ensure we don't hit default/old values */\n> +\n> +\tstartValues.set(controls::ExposureTime, 15000);\n> +\tstartValues.set(controls::AnalogueGain, 4.0);\n> +\n> +\tauto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> +\trunCaptureSession();\n> +\n> +\tEXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> +\tEXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> +\n> +\t/* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> +\tdouble brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> +\tEXPECT_GT(brightnessChange, 2.0);\n> +}\n> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> new file mode 100644\n> index 00000000..e783f024\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> @@ -0,0 +1,41 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.h - Tests for per frame controls\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \n> +\n> +#include \"../common/event_loop.h\"\n> +\n> +#include \"simple_capture.h\"\n> +#include \"time_sheet.h\"\n> +\n> +class PerFrameControls : public SimpleCapture\n> +{\n> +public:\n> +\tPerFrameControls(std::shared_ptr camera);\n> +\n> +\tstd::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> +\tvoid runCaptureSession();\n> +\n> +\tvoid testFramePreciseExposureChange();\n> +\tvoid testFramePreciseGainChange();\n> +\tvoid testExposureGainIsAppliedOnFirstFrame();\n> +\tvoid testExposureGainFromFirstRequestGetsApplied();\n> +\tvoid testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +\n> +\tint queueRequest(libcamera::Request *request);\n> +\tvoid requestComplete(libcamera::Request *request) override;\n> +\n> +\tunsigned int queueCount_;\n> +\tunsigned int captureCount_;\n> +\tunsigned int captureLimit_;\n> +\n> +\tstd::weak_ptr timeSheet_;\n> +};\n> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> index cf4d7cf3..56680a83 100644\n> --- a/src/apps/lc-compliance/simple_capture.cpp\n> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> \t}\n> }\n> \n> -void SimpleCapture::start()\n> +void SimpleCapture::start(const ControlList *controls)\n> {\n> \tStream *stream = config_->at(0).stream();\n> \tint count = allocator_->allocate(stream);\n> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n> \n> \tcamera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> \n> -\tASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> +\tASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> }\n> \n> void SimpleCapture::stop()\n> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> index 2911d601..54b1d54b 100644\n> --- a/src/apps/lc-compliance/simple_capture.h\n> +++ b/src/apps/lc-compliance/simple_capture.h\n> @@ -22,7 +22,7 @@ protected:\n> \tSimpleCapture(std::shared_ptr camera);\n> \tvirtual ~SimpleCapture();\n> \n> -\tvoid start();\n> +\tvoid start(const libcamera::ControlList *controls = nullptr);\n> \tvoid stop();\n> \n> \tvirtual void requestComplete(libcamera::Request *request) = 0;\n> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> new file mode 100644\n> index 00000000..9a0e6544\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> @@ -0,0 +1,135 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.cpp\n> + */\n> +#include \"time_sheet.h\"\n> +\n> +#include \n> +#include \n> +\n> +#include \"libcamera/internal/formats.h\"\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +using namespace libcamera;\n> +\n> +double calcPixelMeanNV12(const uint8_t *data)\n> +{\n> +\treturn (double)*data;\n> +}\n> +\n> +double calcPixelMeanRAW10(const uint8_t *data)\n> +{\n> +\treturn (double)*((const uint16_t *)data);\n> +}\n> +\n> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> +{\n> +\tconst Request::BufferMap &buffers = request->buffers();\n> +\tfor (const auto &[stream, buffer] : buffers) {\n> +\t\tMappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> +\t\tif (in.isValid()) {\n> +\t\t\tauto data = in.planes()[0].data();\n> +\t\t\tauto streamConfig = stream->configuration();\n> +\t\t\tauto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> +\n> +\t\t\tstd::function calcPixelMean;\n> +\t\t\tint pixelStride;\n> +\n> +\t\t\tswitch (streamConfig.pixelFormat) {\n> +\t\t\tcase formats::NV12:\n> +\t\t\t\tcalcPixelMean = calcPixelMeanNV12;\n> +\t\t\t\tpixelStride = 1;\n> +\t\t\t\tbreak;\n> +\t\t\tcase formats::SRGGB10:\n> +\t\t\t\tcalcPixelMean = calcPixelMeanRAW10;\n> +\t\t\t\tpixelStride = 2;\n> +\t\t\t\tbreak;\n> +\t\t\tdefault:\n> +\t\t\t\tstd::stringstream s;\n> +\t\t\t\ts << \"Unsupported Pixelformat \" << formatInfo.name;\n> +\t\t\t\tthrow std::invalid_argument(s.str());\n> +\t\t\t}\n> +\n> +\t\t\tdouble sum = 0;\n> +\t\t\tint w = 20;\n> +\t\t\tint xs = streamConfig.size.width / 2 - w / 2;\n> +\t\t\tint ys = streamConfig.size.height / 2 - w / 2;\n> +\n> +\t\t\tfor (auto y = ys; y < ys + w; y++) {\n> +\t\t\t\tauto line = data + y * streamConfig.stride;\n> +\t\t\t\tfor (auto x = xs; x < xs + w; x++) {\n> +\t\t\t\t\tsum += calcPixelMean(line + x * pixelStride);\n> +\t\t\t\t}\n> +\t\t\t}\n> +\t\t\tsum = sum / (w * w);\n> +\t\t\treturn sum;\n> +\t\t}\n> +\t}\n> +\treturn 0;\n> +}\n> +\n> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> +\t: controls_(idmap)\n> +{\n> +}\n> +\n> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> +{\n> +\tmetadata_ = request->metadata();\n> +\n> +\tspotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> +\tif (previous) {\n> +\t\tbrightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> +\t}\n> +\tsequence_ = request->sequence();\n> +}\n> +\n> +void TimeSheetEntry::printInfo()\n> +{\n> +\tstd::cout << \"=== Frame \" << sequence_ << std::endl;\n> +\tstd::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> +\n> +\tif (!metadata_.empty()) {\n> +\t\tstd::cout << \"Metadata:\" << std::endl;\n> +\t\tauto idMap = metadata_.idMap();\n> +\t\tassert(idMap);\n> +\t\tfor (const auto &[id, value] : metadata_) {\n> +\t\t\tstd::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> +\t\t}\n> +\t}\n> +}\n> +\n> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> +{\n> +\tauto &entry = entries_[pos];\n> +\tif (!entry)\n> +\t\tentry = std::make_shared(idmap_);\n> +\treturn *entry;\n> +}\n> +\n> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> +{\n> +\trequest->controls() = get(sequence).controls();\n> +}\n> +\n> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> +{\n> +\tuint32_t sequence = request->sequence();\n> +\tauto &entry = get(sequence);\n> +\tTimeSheetEntry *previous = nullptr;\n> +\tif (sequence >= 1) {\n> +\t\tprevious = entries_[sequence - 1].get();\n> +\t}\n> +\n> +\tentry.handleCompleteRequest(request, previous);\n> +}\n> +\n> +void TimeSheet::printAllInfos()\n> +{\n> +\tfor (auto entry : entries_) {\n> +\t\tif (entry)\n> +\t\t\tentry->printInfo();\n> +\t}\n> +}\n> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> new file mode 100644\n> index 00000000..c155763c\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.h\n> @@ -0,0 +1,53 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.h\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +class TimeSheetEntry\n> +{\n> +public:\n> +\tTimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> +\tTimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> +\tTimeSheetEntry(const TimeSheetEntry &) = delete;\n> +\n> +\tlibcamera::ControlList &controls() { return controls_; };\n> +\tlibcamera::ControlList &metadata() { return metadata_; };\n> +\tvoid handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> +\tvoid printInfo();\n> +\tdouble getSpotBrightness() const { return spotBrightness_; };\n> +\tdouble getBrightnessChange() const { return brightnessChange_; };\n> +\n> +private:\n> +\tdouble spotBrightness_ = 0.0;\n> +\tdouble brightnessChange_ = 0.0;\n> +\tlibcamera::ControlList controls_;\n> +\tlibcamera::ControlList metadata_;\n> +\tuint32_t sequence_ = 0;\n> +};\n> +\n> +class TimeSheet\n> +{\n> +public:\n> +\tTimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> +\t\t: idmap_(idmap), entries_(count){};\n> +\n> +\tvoid prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> +\tvoid handleCompleteRequest(libcamera::Request *request);\n> +\tvoid printAllInfos();\n> +\n> +\tTimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> +\tTimeSheetEntry &get(size_t pos);\n> +\n> +private:\n> +\tconst libcamera::ControlIdMap &idmap_;\n> +\tstd::vector> entries_;\n> +};","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id AB651BD160\n\tfor ;\n\tThu, 29 Feb 2024 17:08:55 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id CCD2662871;\n\tThu, 29 Feb 2024 18:08:54 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DB9D262867\n\tfor ;\n\tThu, 29 Feb 2024 18:08:52 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:69c9:7315:683e:a8b] (unknown\n\t[IPv6:2a00:6020:448c:6c00:69c9:7315:683e:a8b])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 22D57C67\n\tfor ;\n\tThu, 29 Feb 2024 18:08:39 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"BvUa49fE\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709226519;\n\tbh=fmN29/P0g5y4/9wD/3g7BuJofBqn3d5hXHvYccqEO8Y=;\n\th=Date:Subject:To:References:From:In-Reply-To:From;\n\tb=BvUa49fEQK3LU0q9H89eeDEC2XMiHAL0aHetyqE+uAmk1PBEOTFB0sEu2UPYtjobz\n\tTXyUwnElkuXZExae/oBV5Rara4kqaoZaSJ34wQmofYifKppk0qKeH5IIgdqNgFhcjH\n\t5uM238XE0WUOCHPRqFXC3HgdM7fnP9xgUamBWsQc=","Message-ID":"<9810ad7e-02a7-4e86-848b-fdeffe610ec3@ideasonboard.com>","Date":"Thu, 29 Feb 2024 18:08:52 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"libcamera-devel@lists.libcamera.org","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","From":"Stefan Klug ","In-Reply-To":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28806,"web_url":"https://patchwork.libcamera.org/comment/28806/","msgid":"<170922817185.1011926.7574189277331671166@ping.linuxembedded.co.uk>","date":"2024-02-29T17:36:11","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Hi Stefan,\n\nSome quick review notes.\n\nI'm happy to see a test that actually tries to validate the\nDelayedControls implementations, and considers the real effects on the\nsensor.\n\nWe try to break patches down to single topic commits, so here I can see\nat least\n - The introduction of the TimeSheet helper class\n - Fix the SimpleCapture::start() to accept a control list\n - Add per-frame-control tests.\n\nQuoting Stefan Klug (2024-02-29 17:01:15)\n> These tests check if controls (only exposure time and analogue gain at\n> the moment) get applied on the frame they were requested for.\n> \n> This is tested by looking at the metadata and additionally by\n> calculating a mean brightness on a centered rect of 20x20 pixels.\n> \n> Until today, these tests where only run on a project specific branch\n> with a modified simple pipeline. In theory they should pass on a\n> current master :-)\n\nI love good theories! ;-) I'm curious to see what happens on master and\nother platforms.\n\n\n> Current test setup: imx219 with simple pipeline on an imx8mp.\n> Modifications of either the exposure delay or the gain delay in\n> the camera_sensor class resulted in test failures.\n> Which is exactly what this test shall proove.\n\n'prove' ;-)\n\n> \n> Signed-off-by: Stefan Klug \n> ---\n> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> src/apps/lc-compliance/meson.build | 2 +\n> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> src/apps/lc-compliance/simple_capture.h | 2 +-\n> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> 8 files changed, 487 insertions(+), 3 deletions(-)\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> create mode 100644 src/apps/lc-compliance/time_sheet.h\n> \n> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> index 1dcfcf92..43fe59f3 100644\n> --- a/src/apps/lc-compliance/capture_test.cpp\n> +++ b/src/apps/lc-compliance/capture_test.cpp\n> @@ -11,6 +11,7 @@\n> #include \n> \n> #include \"environment.h\"\n> +#include \"per_frame_controls.h\"\n> #include \"simple_capture.h\"\n> \n> using namespace libcamera;\n> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> testing::Combine(testing::ValuesIn(ROLES),\n> testing::ValuesIn(NUMREQUESTS)),\n> SingleStream::nameParameters);\n> +\n> +/*\n> + * Test Per frame controls\n> + */\n> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testFramePreciseExposureChange();\n> +}\n> +\n> +TEST_F(SingleStream, testFramePreciseGainChange)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testFramePreciseGainChange();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainIsAppliedOnFirstFrame();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainFromFirstRequestGetsApplied();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +}\n> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> index c792f072..2a6f52af 100644\n> --- a/src/apps/lc-compliance/meson.build\n> +++ b/src/apps/lc-compliance/meson.build\n> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> 'capture_test.cpp',\n> 'environment.cpp',\n> 'main.cpp',\n> + 'per_frame_controls.cpp',\n> 'simple_capture.cpp',\n> + 'time_sheet.cpp',\n> ])\n> \n> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> new file mode 100644\n> index 00000000..70fc44ac\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> @@ -0,0 +1,214 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.cpp - Tests for per frame controls\n> + */\n> +#include \"per_frame_controls.h\"\n> +\n> +#include \n> +\n> +#include \"time_sheet.h\"\n> +\n> +using namespace libcamera;\n> +\n> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> + : SimpleCapture(camera)\n> +{\n> +}\n> +\n> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> +{\n> + ControlList ctrls(camera_->controls().idmap());\n> + /* Ensure defined default values */\n> + ctrls.set(controls::AeEnable, false);\n> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> + ctrls.set(controls::ExposureTime, 10000);\n> + ctrls.set(controls::AnalogueGain, 1.0);\n> +\n> + if (controls) {\n> + ctrls.merge(*controls, true);\n> + }\n\nNo need for { } on a single line statement.\n\tif (controls)\n\t\tctrls.merge(*controls, true);\n\nis sufficient in our code style.\n\n\n> +\n> + start(&ctrls);\n> +\n> + queueCount_ = 0;\n> + captureCount_ = 0;\n> + captureLimit_ = framesToCapture;\n> +\n> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> + timeSheet_ = timeSheet;\n\nI'm curious. Why shared and not make_unique? Is it kept / shared in\nmultiple locations? I guess I'll see later.\n\n> + return timeSheet;\n> +}\n> +\n> +int PerFrameControls::queueRequest(Request *request)\n> +{\n> + queueCount_++;\n> + if (queueCount_ > captureLimit_)\n\nShould this return an error as it's /not/ queueing? Maybe not essential\n... lets see how it's used...\n\n\nProbably not as we just want to stop queueing and we still want to wait\nfor the requests queued to complete...\n\n\n> + return 0;\n> +\n> + auto ts = timeSheet_.lock();\n> + if (ts) {\n> + ts->prepareForQueue(request, queueCount_ - 1);\n> + }\n\nSingle line so no braces, same elsewhere.\n\n> +\n> + return camera_->queueRequest(request);\n> +}\n> +\n> +void PerFrameControls::requestComplete(Request *request)\n> +{\n> + auto ts = timeSheet_.lock();\n> + if (ts) {\n> + ts->handleCompleteRequest(request);\n> + }\n> +\n> + captureCount_++;\n> + if (captureCount_ >= captureLimit_) {\n> + loop_->exit(0);\n> + return;\n> + }\n> +\n> + request->reuse(Request::ReuseBuffers);\n> + if (queueRequest(request))\n> + loop_->exit(-EINVAL);\n> +}\n> +\n> +void PerFrameControls::runCaptureSession()\n> +{\n> + Stream *stream = config_->at(0).stream();\n> + const std::vector> &buffers = allocator_->buffers(stream);\n> +\n> + /* Queue the recommended number of reqeuests. */\n> + for (const std::unique_ptr &buffer : buffers) {\n> + std::unique_ptr request = camera_->createRequest();\n> + request->addBuffer(stream, buffer.get());\n> + queueRequest(request.get());\n> + requests_.push_back(std::move(request));\n> + }\n> +\n> + /* Run capture session. */\n> + loop_ = new EventLoop();\n> + loop_->exec();\n> + stop();\n> + delete loop_;\n> +}\n> +\n> +void PerFrameControls::testFramePreciseExposureChange()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(10);\n> + auto &ts = *timeSheet;\n> +\n> +\n> + ts[3].controls().set(controls::ExposureTime, 5000);\n> + //wait a few frames to settle\n\n\t/* Single line comment style. */\n\n> + ts[6].controls().set(controls::ExposureTime, 20000);\n> + ts.printAllInfos();\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> +\n> + /* No increase just before setting exposure */\n> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n\nI'd wrap this a little, but it's better to keep the comment in one.\nPerhaps:\n\n\tEXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05)\n\t\t<< \"Brightness changed too much before the expected time of change (control delay too high?).\";\n\n> + /*\n> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> + * This should be investigated.\n> + */\n\nindentation is broken here. Also lines could be wrapped to target\n80chars.\n\n\n\n> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> + /* No increase just after setting exposure */\n> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +\n> + /* No increase just after setting exposure */\n> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n\nSame here, I'd wrap the message to the line below. But that's all we can\ndo here I think.\n\n\"changed too much2 frames\" probably needs cleaning up.\n\n> +}\n> +\n> +void PerFrameControls::testFramePreciseGainChange()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(10);\n> + auto &ts = *timeSheet;\n> +\n> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n> + //wait a few frames to settle\n> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +\n> + /* No increase just before setting gain */\n> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> + /*\n> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> + */\n\nSame indentation issues here, and wrapping where possible.\n\n> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> + /* No increase just after setting gain */\n> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(5);\n> + auto &ts = *timeSheet;\n> +\n> + ts[0].controls().set(controls::ExposureTime, 10000);\n> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + /* We expect it to be applied after 3 frames, the latest*/\n\n\"latest */\"\n\n> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(5);\n> + auto &ts = *timeSheet;\n> +\n> + ts[0].controls().set(controls::ExposureTime, 8000);\n> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n> + ts[1].controls().set(controls::ExposureTime, 10000);\n> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + /* We expect it to be applied after 3 frames, the latest*/\n\n\"latest */\"\n\n> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> +{\n> + ControlList startValues;\n> + startValues.set(controls::ExposureTime, 5000);\n> + startValues.set(controls::AnalogueGain, 1.0);\n> +\n> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> +\n> + /* Second capture with different values to ensure we don't hit default/old values */\n> +\n> + startValues.set(controls::ExposureTime, 15000);\n> + startValues.set(controls::AnalogueGain, 4.0);\n> +\n> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> +\n> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> + EXPECT_GT(brightnessChange, 2.0);\n> +}\n> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> new file mode 100644\n> index 00000000..e783f024\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> @@ -0,0 +1,41 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.h - Tests for per frame controls\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \n> +\n> +#include \"../common/event_loop.h\"\n> +\n> +#include \"simple_capture.h\"\n> +#include \"time_sheet.h\"\n> +\n> +class PerFrameControls : public SimpleCapture\n> +{\n> +public:\n> + PerFrameControls(std::shared_ptr camera);\n> +\n> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> + void runCaptureSession();\n> +\n> + void testFramePreciseExposureChange();\n> + void testFramePreciseGainChange();\n> + void testExposureGainIsAppliedOnFirstFrame();\n> + void testExposureGainFromFirstRequestGetsApplied();\n> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +\n> + int queueRequest(libcamera::Request *request);\n> + void requestComplete(libcamera::Request *request) override;\n> +\n> + unsigned int queueCount_;\n> + unsigned int captureCount_;\n> + unsigned int captureLimit_;\n> +\n> + std::weak_ptr timeSheet_;\n> +};\n> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> index cf4d7cf3..56680a83 100644\n> --- a/src/apps/lc-compliance/simple_capture.cpp\n> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> }\n> }\n> \n> -void SimpleCapture::start()\n> +void SimpleCapture::start(const ControlList *controls)\n> {\n> Stream *stream = config_->at(0).stream();\n> int count = allocator_->allocate(stream);\n> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n> \n> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> \n> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> }\n> \n> void SimpleCapture::stop()\n> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> index 2911d601..54b1d54b 100644\n> --- a/src/apps/lc-compliance/simple_capture.h\n> +++ b/src/apps/lc-compliance/simple_capture.h\n> @@ -22,7 +22,7 @@ protected:\n> SimpleCapture(std::shared_ptr camera);\n> virtual ~SimpleCapture();\n> \n> - void start();\n> + void start(const libcamera::ControlList *controls = nullptr);\n> void stop();\n\n\nI would suggest breaking out the change here that enables setting\ncontrols at SimpleCapture::start() to it's own patch.\n\n\n> \n> virtual void requestComplete(libcamera::Request *request) = 0;\n> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> new file mode 100644\n> index 00000000..9a0e6544\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.cpp\n\nI think introducting TimeSheet can be a separate patch to make it\nclearer.\n\n\n> @@ -0,0 +1,135 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.cpp\n> + */\n> +#include \"time_sheet.h\"\n> +\n> +#include \n> +#include \n> +\n> +#include \"libcamera/internal/formats.h\"\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +using namespace libcamera;\n> +\n> +double calcPixelMeanNV12(const uint8_t *data)\n> +{\n> + return (double)*data;\n\nOh, from the name \"calcPixelMean\" I was expecting a calculation, not\njust a cast.\n\n\nMaybe \"double readPixelNV12()\" or something.\n\nI think Laurent started looking at some 'image' type classes to tackle\nsomething similar to this problem.\n\n\n> +}\n> +\n> +double calcPixelMeanRAW10(const uint8_t *data)\n> +{\n> + return (double)*((const uint16_t *)data);\n> +}\n> +\n> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> +{\n> + const Request::BufferMap &buffers = request->buffers();\n> + for (const auto &[stream, buffer] : buffers) {\n> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> + if (in.isValid()) {\n> + auto data = in.planes()[0].data();\n> + auto streamConfig = stream->configuration();\n> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> +\n> + std::function calcPixelMean;\n> + int pixelStride;\n> +\n> + switch (streamConfig.pixelFormat) {\n> + case formats::NV12:\n> + calcPixelMean = calcPixelMeanNV12;\n> + pixelStride = 1;\n> + break;\n> + case formats::SRGGB10:\n> + calcPixelMean = calcPixelMeanRAW10;\n> + pixelStride = 2;\n> + break;\n> + default:\n> + std::stringstream s;\n> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n> + throw std::invalid_argument(s.str());\n> + }\n> +\n> + double sum = 0;\n> + int w = 20;\n> + int xs = streamConfig.size.width / 2 - w / 2;\n> + int ys = streamConfig.size.height / 2 - w / 2;\n> +\n> + for (auto y = ys; y < ys + w; y++) {\n> + auto line = data + y * streamConfig.stride;\n> + for (auto x = xs; x < xs + w; x++) {\n> + sum += calcPixelMean(line + x * pixelStride);\n> + }\n> + }\n> + sum = sum / (w * w);\n> + return sum;\n> + }\n> + }\n> + return 0;\n> +}\n> +\n> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> + : controls_(idmap)\n> +{\n> +}\n> +\n> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> +{\n> + metadata_ = request->metadata();\n> +\n> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> + if (previous) {\n> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> + }\n> + sequence_ = request->sequence();\n> +}\n> +\n> +void TimeSheetEntry::printInfo()\n> +{\n> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> +\n> + if (!metadata_.empty()) {\n> + std::cout << \"Metadata:\" << std::endl;\n> + auto idMap = metadata_.idMap();\n> + assert(idMap);\n> + for (const auto &[id, value] : metadata_) {\n> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> + }\n> + }\n> +}\n> +\n> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> +{\n> + auto &entry = entries_[pos];\n> + if (!entry)\n> + entry = std::make_shared(idmap_);\n> + return *entry;\n> +}\n> +\n> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> +{\n> + request->controls() = get(sequence).controls();\n> +}\n> +\n> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> +{\n> + uint32_t sequence = request->sequence();\n> + auto &entry = get(sequence);\n> + TimeSheetEntry *previous = nullptr;\n> + if (sequence >= 1) {\n> + previous = entries_[sequence - 1].get();\n> + }\n> +\n> + entry.handleCompleteRequest(request, previous);\n> +}\n> +\n> +void TimeSheet::printAllInfos()\n> +{\n> + for (auto entry : entries_) {\n> + if (entry)\n> + entry->printInfo();\n> + }\n> +}\n> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> new file mode 100644\n> index 00000000..c155763c\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.h\n> @@ -0,0 +1,53 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.h\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +class TimeSheetEntry\n> +{\n> +public:\n> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n> +\n> + libcamera::ControlList &controls() { return controls_; };\n> + libcamera::ControlList &metadata() { return metadata_; };\n> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> + void printInfo();\n> + double getSpotBrightness() const { return spotBrightness_; };\n> + double getBrightnessChange() const { return brightnessChange_; };\n> +\n> +private:\n> + double spotBrightness_ = 0.0;\n> + double brightnessChange_ = 0.0;\n> + libcamera::ControlList controls_;\n> + libcamera::ControlList metadata_;\n> + uint32_t sequence_ = 0;\n> +};\n> +\n> +class TimeSheet\n> +{\n> +public:\n> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> + : idmap_(idmap), entries_(count){};\n> +\n> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> + void handleCompleteRequest(libcamera::Request *request);\n> + void printAllInfos();\n> +\n> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> + TimeSheetEntry &get(size_t pos);\n> +\n> +private:\n> + const libcamera::ControlIdMap &idmap_;\n> + std::vector> entries_;\n> +};\n> -- \n> 2.40.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 9ACC1BE080\n\tfor ;\n\tThu, 29 Feb 2024 17:36:17 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C917B6286C;\n\tThu, 29 Feb 2024 18:36:16 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id F21E461C96\n\tfor ;\n\tThu, 29 Feb 2024 18:36:14 +0100 (CET)","from pendragon.ideasonboard.com\n\t(aztw-30-b2-v4wan-166917-cust845.vm26.cable.virginm.net\n\t[82.37.23.78])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 43B72C67;\n\tThu, 29 Feb 2024 18:36:01 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"s+fkY9gQ\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709228161;\n\tbh=Gfv/Drj8IP+9tmxM+XXouaydtEhqrAMCp9ZL0L+XlTA=;\n\th=In-Reply-To:References:Subject:From:To:Date:From;\n\tb=s+fkY9gQwlXHZv6UbC9w73FLS82Vl+hOna8R/0dYNvpb7/zUv68ZPkNt6ZIWgQ2+D\n\t9YWo1a8ZzqPhsY4jTh41i7GVgY6bd+3C7ju9Yaj9JmDEfTqk2cCJPkjxgMLfo8OVHV\n\t65tG/xSUy/865RxV1jBwDNlQV0mjGxnoLMhapTpo=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","From":"Kieran Bingham ","To":"Stefan Klug ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Thu, 29 Feb 2024 17:36:11 +0000","Message-ID":"<170922817185.1011926.7574189277331671166@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28807,"web_url":"https://patchwork.libcamera.org/comment/28807/","msgid":"","date":"2024-03-01T07:56:34","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Stefan\n\nThanks for posting this and re-starting the discussion of per-frame controls.\n\nCould you perhaps just summarise exactly what you mean by per-frame\ncontrols, that is to say, what is being tested here?\n\nEye-balling the code, I think I understood that:\n\n* You are setting controls in a specific request.\n* When that request completes, you are expecting the controls to have\ntaken effect in the images that were returned with that request (but\nnot earlier).\n\nBut I wasn't totally sure - have I understood that correctly?\n\nAlso, do we know if any other pipeline handlers implement this\nbehaviour? Do folks think that all pipeline handlers should implement\nthis behaviour?\n\nThanks again!\nDavid\n\n\nOn Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n>\n> These tests check if controls (only exposure time and analogue gain at\n> the moment) get applied on the frame they were requested for.\n>\n> This is tested by looking at the metadata and additionally by\n> calculating a mean brightness on a centered rect of 20x20 pixels.\n>\n> Until today, these tests where only run on a project specific branch\n> with a modified simple pipeline. In theory they should pass on a\n> current master :-)\n>\n> Current test setup: imx219 with simple pipeline on an imx8mp.\n> Modifications of either the exposure delay or the gain delay in\n> the camera_sensor class resulted in test failures.\n> Which is exactly what this test shall proove.\n>\n> Signed-off-by: Stefan Klug \n> ---\n> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> src/apps/lc-compliance/meson.build | 2 +\n> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> src/apps/lc-compliance/simple_capture.h | 2 +-\n> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> 8 files changed, 487 insertions(+), 3 deletions(-)\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>\n> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> index 1dcfcf92..43fe59f3 100644\n> --- a/src/apps/lc-compliance/capture_test.cpp\n> +++ b/src/apps/lc-compliance/capture_test.cpp\n> @@ -11,6 +11,7 @@\n> #include \n>\n> #include \"environment.h\"\n> +#include \"per_frame_controls.h\"\n> #include \"simple_capture.h\"\n>\n> using namespace libcamera;\n> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> testing::Combine(testing::ValuesIn(ROLES),\n> testing::ValuesIn(NUMREQUESTS)),\n> SingleStream::nameParameters);\n> +\n> +/*\n> + * Test Per frame controls\n> + */\n> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testFramePreciseExposureChange();\n> +}\n> +\n> +TEST_F(SingleStream, testFramePreciseGainChange)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testFramePreciseGainChange();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainIsAppliedOnFirstFrame();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainFromFirstRequestGetsApplied();\n> +}\n> +\n> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> +{\n> + PerFrameControls capture(camera_);\n> + capture.configure(StreamRole::Viewfinder);\n> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +}\n> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> index c792f072..2a6f52af 100644\n> --- a/src/apps/lc-compliance/meson.build\n> +++ b/src/apps/lc-compliance/meson.build\n> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> 'capture_test.cpp',\n> 'environment.cpp',\n> 'main.cpp',\n> + 'per_frame_controls.cpp',\n> 'simple_capture.cpp',\n> + 'time_sheet.cpp',\n> ])\n>\n> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> new file mode 100644\n> index 00000000..70fc44ac\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> @@ -0,0 +1,214 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.cpp - Tests for per frame controls\n> + */\n> +#include \"per_frame_controls.h\"\n> +\n> +#include \n> +\n> +#include \"time_sheet.h\"\n> +\n> +using namespace libcamera;\n> +\n> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> + : SimpleCapture(camera)\n> +{\n> +}\n> +\n> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> +{\n> + ControlList ctrls(camera_->controls().idmap());\n> + /* Ensure defined default values */\n> + ctrls.set(controls::AeEnable, false);\n> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> + ctrls.set(controls::ExposureTime, 10000);\n> + ctrls.set(controls::AnalogueGain, 1.0);\n> +\n> + if (controls) {\n> + ctrls.merge(*controls, true);\n> + }\n> +\n> + start(&ctrls);\n> +\n> + queueCount_ = 0;\n> + captureCount_ = 0;\n> + captureLimit_ = framesToCapture;\n> +\n> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> + timeSheet_ = timeSheet;\n> + return timeSheet;\n> +}\n> +\n> +int PerFrameControls::queueRequest(Request *request)\n> +{\n> + queueCount_++;\n> + if (queueCount_ > captureLimit_)\n> + return 0;\n> +\n> + auto ts = timeSheet_.lock();\n> + if (ts) {\n> + ts->prepareForQueue(request, queueCount_ - 1);\n> + }\n> +\n> + return camera_->queueRequest(request);\n> +}\n> +\n> +void PerFrameControls::requestComplete(Request *request)\n> +{\n> + auto ts = timeSheet_.lock();\n> + if (ts) {\n> + ts->handleCompleteRequest(request);\n> + }\n> +\n> + captureCount_++;\n> + if (captureCount_ >= captureLimit_) {\n> + loop_->exit(0);\n> + return;\n> + }\n> +\n> + request->reuse(Request::ReuseBuffers);\n> + if (queueRequest(request))\n> + loop_->exit(-EINVAL);\n> +}\n> +\n> +void PerFrameControls::runCaptureSession()\n> +{\n> + Stream *stream = config_->at(0).stream();\n> + const std::vector> &buffers = allocator_->buffers(stream);\n> +\n> + /* Queue the recommended number of reqeuests. */\n> + for (const std::unique_ptr &buffer : buffers) {\n> + std::unique_ptr request = camera_->createRequest();\n> + request->addBuffer(stream, buffer.get());\n> + queueRequest(request.get());\n> + requests_.push_back(std::move(request));\n> + }\n> +\n> + /* Run capture session. */\n> + loop_ = new EventLoop();\n> + loop_->exec();\n> + stop();\n> + delete loop_;\n> +}\n> +\n> +void PerFrameControls::testFramePreciseExposureChange()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(10);\n> + auto &ts = *timeSheet;\n> +\n> +\n> + ts[3].controls().set(controls::ExposureTime, 5000);\n> + //wait a few frames to settle\n> + ts[6].controls().set(controls::ExposureTime, 20000);\n> + ts.printAllInfos();\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> +\n> + /* No increase just before setting exposure */\n> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> + /*\n> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> + * This should be investigated.\n> + */\n> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> + /* No increase just after setting exposure */\n> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +\n> + /* No increase just after setting exposure */\n> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> +}\n> +\n> +void PerFrameControls::testFramePreciseGainChange()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(10);\n> + auto &ts = *timeSheet;\n> +\n> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n> + //wait a few frames to settle\n> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +\n> + /* No increase just before setting gain */\n> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> + /*\n> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> + */\n> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> +\n> + /* No increase just after setting gain */\n> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(5);\n> + auto &ts = *timeSheet;\n> +\n> + ts[0].controls().set(controls::ExposureTime, 10000);\n> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + /* We expect it to be applied after 3 frames, the latest*/\n> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> +{\n> + auto timeSheet = startCaptureWithTimeSheet(5);\n> + auto &ts = *timeSheet;\n> +\n> + ts[0].controls().set(controls::ExposureTime, 8000);\n> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n> + ts[1].controls().set(controls::ExposureTime, 10000);\n> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n> +\n> + runCaptureSession();\n> +\n> + /* We expect it to be applied after 3 frames, the latest*/\n> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> +}\n> +\n> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> +{\n> + ControlList startValues;\n> + startValues.set(controls::ExposureTime, 5000);\n> + startValues.set(controls::AnalogueGain, 1.0);\n> +\n> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> +\n> + /* Second capture with different values to ensure we don't hit default/old values */\n> +\n> + startValues.set(controls::ExposureTime, 15000);\n> + startValues.set(controls::AnalogueGain, 4.0);\n> +\n> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> +\n> + runCaptureSession();\n> +\n> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> +\n> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> + EXPECT_GT(brightnessChange, 2.0);\n> +}\n> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> new file mode 100644\n> index 00000000..e783f024\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> @@ -0,0 +1,41 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * per_frame_controls.h - Tests for per frame controls\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \n> +\n> +#include \"../common/event_loop.h\"\n> +\n> +#include \"simple_capture.h\"\n> +#include \"time_sheet.h\"\n> +\n> +class PerFrameControls : public SimpleCapture\n> +{\n> +public:\n> + PerFrameControls(std::shared_ptr camera);\n> +\n> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> + void runCaptureSession();\n> +\n> + void testFramePreciseExposureChange();\n> + void testFramePreciseGainChange();\n> + void testExposureGainIsAppliedOnFirstFrame();\n> + void testExposureGainFromFirstRequestGetsApplied();\n> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n> +\n> + int queueRequest(libcamera::Request *request);\n> + void requestComplete(libcamera::Request *request) override;\n> +\n> + unsigned int queueCount_;\n> + unsigned int captureCount_;\n> + unsigned int captureLimit_;\n> +\n> + std::weak_ptr timeSheet_;\n> +};\n> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> index cf4d7cf3..56680a83 100644\n> --- a/src/apps/lc-compliance/simple_capture.cpp\n> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> }\n> }\n>\n> -void SimpleCapture::start()\n> +void SimpleCapture::start(const ControlList *controls)\n> {\n> Stream *stream = config_->at(0).stream();\n> int count = allocator_->allocate(stream);\n> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>\n> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>\n> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> }\n>\n> void SimpleCapture::stop()\n> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> index 2911d601..54b1d54b 100644\n> --- a/src/apps/lc-compliance/simple_capture.h\n> +++ b/src/apps/lc-compliance/simple_capture.h\n> @@ -22,7 +22,7 @@ protected:\n> SimpleCapture(std::shared_ptr camera);\n> virtual ~SimpleCapture();\n>\n> - void start();\n> + void start(const libcamera::ControlList *controls = nullptr);\n> void stop();\n>\n> virtual void requestComplete(libcamera::Request *request) = 0;\n> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> new file mode 100644\n> index 00000000..9a0e6544\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> @@ -0,0 +1,135 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.cpp\n> + */\n> +#include \"time_sheet.h\"\n> +\n> +#include \n> +#include \n> +\n> +#include \"libcamera/internal/formats.h\"\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +using namespace libcamera;\n> +\n> +double calcPixelMeanNV12(const uint8_t *data)\n> +{\n> + return (double)*data;\n> +}\n> +\n> +double calcPixelMeanRAW10(const uint8_t *data)\n> +{\n> + return (double)*((const uint16_t *)data);\n> +}\n> +\n> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> +{\n> + const Request::BufferMap &buffers = request->buffers();\n> + for (const auto &[stream, buffer] : buffers) {\n> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> + if (in.isValid()) {\n> + auto data = in.planes()[0].data();\n> + auto streamConfig = stream->configuration();\n> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> +\n> + std::function calcPixelMean;\n> + int pixelStride;\n> +\n> + switch (streamConfig.pixelFormat) {\n> + case formats::NV12:\n> + calcPixelMean = calcPixelMeanNV12;\n> + pixelStride = 1;\n> + break;\n> + case formats::SRGGB10:\n> + calcPixelMean = calcPixelMeanRAW10;\n> + pixelStride = 2;\n> + break;\n> + default:\n> + std::stringstream s;\n> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n> + throw std::invalid_argument(s.str());\n> + }\n> +\n> + double sum = 0;\n> + int w = 20;\n> + int xs = streamConfig.size.width / 2 - w / 2;\n> + int ys = streamConfig.size.height / 2 - w / 2;\n> +\n> + for (auto y = ys; y < ys + w; y++) {\n> + auto line = data + y * streamConfig.stride;\n> + for (auto x = xs; x < xs + w; x++) {\n> + sum += calcPixelMean(line + x * pixelStride);\n> + }\n> + }\n> + sum = sum / (w * w);\n> + return sum;\n> + }\n> + }\n> + return 0;\n> +}\n> +\n> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> + : controls_(idmap)\n> +{\n> +}\n> +\n> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> +{\n> + metadata_ = request->metadata();\n> +\n> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> + if (previous) {\n> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> + }\n> + sequence_ = request->sequence();\n> +}\n> +\n> +void TimeSheetEntry::printInfo()\n> +{\n> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> +\n> + if (!metadata_.empty()) {\n> + std::cout << \"Metadata:\" << std::endl;\n> + auto idMap = metadata_.idMap();\n> + assert(idMap);\n> + for (const auto &[id, value] : metadata_) {\n> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> + }\n> + }\n> +}\n> +\n> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> +{\n> + auto &entry = entries_[pos];\n> + if (!entry)\n> + entry = std::make_shared(idmap_);\n> + return *entry;\n> +}\n> +\n> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> +{\n> + request->controls() = get(sequence).controls();\n> +}\n> +\n> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> +{\n> + uint32_t sequence = request->sequence();\n> + auto &entry = get(sequence);\n> + TimeSheetEntry *previous = nullptr;\n> + if (sequence >= 1) {\n> + previous = entries_[sequence - 1].get();\n> + }\n> +\n> + entry.handleCompleteRequest(request, previous);\n> +}\n> +\n> +void TimeSheet::printAllInfos()\n> +{\n> + for (auto entry : entries_) {\n> + if (entry)\n> + entry->printInfo();\n> + }\n> +}\n> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> new file mode 100644\n> index 00000000..c155763c\n> --- /dev/null\n> +++ b/src/apps/lc-compliance/time_sheet.h\n> @@ -0,0 +1,53 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Ideas on Board Oy\n> + *\n> + * time_sheet.h\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +class TimeSheetEntry\n> +{\n> +public:\n> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n> +\n> + libcamera::ControlList &controls() { return controls_; };\n> + libcamera::ControlList &metadata() { return metadata_; };\n> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> + void printInfo();\n> + double getSpotBrightness() const { return spotBrightness_; };\n> + double getBrightnessChange() const { return brightnessChange_; };\n> +\n> +private:\n> + double spotBrightness_ = 0.0;\n> + double brightnessChange_ = 0.0;\n> + libcamera::ControlList controls_;\n> + libcamera::ControlList metadata_;\n> + uint32_t sequence_ = 0;\n> +};\n> +\n> +class TimeSheet\n> +{\n> +public:\n> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> + : idmap_(idmap), entries_(count){};\n> +\n> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> + void handleCompleteRequest(libcamera::Request *request);\n> + void printAllInfos();\n> +\n> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> + TimeSheetEntry &get(size_t pos);\n> +\n> +private:\n> + const libcamera::ControlIdMap &idmap_;\n> + std::vector> entries_;\n> +};\n> --\n> 2.40.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id CBFCABE080\n\tfor ;\n\tFri, 1 Mar 2024 07:56:49 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1FF606286C;\n\tFri, 1 Mar 2024 08:56:49 +0100 (CET)","from mail-qt1-x833.google.com (mail-qt1-x833.google.com\n\t[IPv6:2607:f8b0:4864:20::833])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DFCAF61C8D\n\tfor ;\n\tFri, 1 Mar 2024 08:56:46 +0100 (CET)","by mail-qt1-x833.google.com with SMTP id\n\td75a77b69052e-42e8a130ebcso19275671cf.1\n\tfor ;\n\tThu, 29 Feb 2024 23:56:46 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"a3diGdk3\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1709279805; x=1709884605;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=58HLHpVafiFKAa4oxlVAsWRSTlYNpEhAfybm0f/zmLE=;\n\tb=a3diGdk3FyUKXgGo/GfjZADYQYqomGzaVdZ0+SbWGU6U7JiQfWeGJDZr5KBijPF7QK\n\tIOPLay7CFDJ246jrm8iVeFszrscoAPNyJ+K+73Lyhkvvr7U5S4xrCwVkhabzc/j4G8gY\n\t0vjs7JIlgU7L38lJolUxiM3HusmILsJvfKZvdmFoE8ZVuwrTMBwjEEwFHwkseUj+ueg4\n\tUSJ5W8ek4PH1Mrzh1s2PFfzTalA1Ntls4cJcA3e9M7JWHaEntTejd1SZ+ZsvZc80BdJC\n\tyPJhrdnLkN8kMZLT7ecFizo0JrKFLvs5A/wnLwIsNLxnKxg5mOn0XGPdagGQ/sKdmYSo\n\tLKTw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1709279805; x=1709884605;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=58HLHpVafiFKAa4oxlVAsWRSTlYNpEhAfybm0f/zmLE=;\n\tb=ZUrooMnKtAFr1yW8x/VqQ2cGp3cVCOioWq8Dul51mcH7gA3tQjIrw9iWtFh68Bh51p\n\tiLbJr0X5A8IhQ3VJQ3w3lrSGINU0g/yaNsyeTDE7luOBF+yFtp6NAK5o4bPNvPcOyJby\n\tIcrlCvPutFT34w+fBKXD9z1V/IMGvQyiVPnM2adLQz6A89I2ygh1HwQDt9bLMHqbv4RI\n\t6N3IhaLsUTeVGWgMahi26Fh/FexJfIqwn3iX6lzbjICAyfGROuuenW/y37MqluULCNuy\n\tUWn57OeCC+tg9PwvJFHH5rJiokLn5jrxshAr3+TLg9pMRGrxr+xAZJiDX/Xw/aBVyQa1\n\t/niA==","X-Gm-Message-State":"AOJu0YxCquKpO5UGiYo5CMl3Lhxi+2fQ9F4zaARWAhlri56r4e5O2PjE\n\tyR9GTxymamKx8o99O7qfPgsv7ajyEUkOS3ac74vJ1dOMqQT+/QTjdi4bqlwu4SE419Fgy97g/fu\n\tpirOvewF1RiTf+SO123CMbUbj2HlnwXAtGJNSpsXFnna8/CEa","X-Google-Smtp-Source":"AGHT+IH3ADwJWdPgTN/Xl06oXy+0l5RjGpeDd3jG66TAC+dFYMXPlrBvTpgQ5eLKavj4T3wAr+tnhHiIn7V0114LpNg=","X-Received":"by 2002:a05:622a:214:b0:42e:8a79:68f0 with SMTP id\n\tb20-20020a05622a021400b0042e8a7968f0mr1284458qtx.23.1709279805413;\n\tThu, 29 Feb 2024 23:56:45 -0800 (PST)","MIME-Version":"1.0","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","In-Reply-To":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>","From":"David Plowman ","Date":"Fri, 1 Mar 2024 07:56:34 +0000","Message-ID":"","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","To":"Stefan Klug ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28810,"web_url":"https://patchwork.libcamera.org/comment/28810/","msgid":"<1e83d77c-7b98-492b-875f-cda17b87256a@ideasonboard.com>","date":"2024-03-01T10:48:36","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi David,\n\nthanks for your comment.\n\nAm 01.03.24 um 08:56 schrieb David Plowman:\n> Hi Stefan\n> \n> Thanks for posting this and re-starting the discussion of per-frame controls.\n> \n> Could you perhaps just summarise exactly what you mean by per-frame\n> controls, that is to say, what is being tested here?\n> \n> Eye-balling the code, I think I understood that:\n> \n> * You are setting controls in a specific request.\n> * When that request completes, you are expecting the controls to have\n> taken effect in the images that were returned with that request (but\n> not earlier).\n> \n> But I wasn't totally sure - have I understood that correctly?\n> \n> Also, do we know if any other pipeline handlers implement this\n> behaviour? Do folks think that all pipeline handlers should implement\n> this behaviour?\n\nYou are completely right. This needs a bit more context.\n\nIt all started on my side with implementing metadata support in \nSimplePipeline handler. I soon hit some corner cases where I expected \nthings to behave differently in cooperation with DelayedControls. So \neither I hit a bug in DelayedControls or I didn't fully understand the \nconcept behind it.\n\nI ended up changing several things in DelayedControls which I believe \nwhere correct. The question then was how to prove correctness as all \ncurrent users of DelayedControls where ISP implementations. There are \nsome unittests, but the order of calls in these tests also felt \ncounterintuitive (might well be, that it's just missing knowledge on my \nside).\n\nIn that area there were also no tests in lc-compliance which tested the \nbehaviour of an actual sensor/isp combination. So I started to write \nthese tests with my personal expectation of how I believe things should \nwork. These tests pass on my SimplePipeline. I also tried to massage the \nrkisp pipeline to pass these tests and hit some corners where work would \nbe required.\n\nBefore digging deeper into that I believe it makes sense to see if my \nexpectations are correct and if a broader audience agrees to them. So \nhere we are, that's the reason for the RFC.\n\nNow on to the technical details:\nYes my expectation on per-frame-controls would be: If I queue something \nfor frame x, I expect that the system tries to fullfill exacty that \nrequest (and never earlier). I'm well aware that this will not be \npossible in every case when an ISP is involved, so the ISP should do the \nbest it can, but the metadata should reflect what was actually achieved. \nAll the tests are currently for manual mode, so no regulation is \ninvolved (at least for the params I test).\n\nDo you agree with these assumptions? Looking forward to you opinion.\n\nBest regards,\nStefan\n\n> \n> Thanks again!\n> David\n> \n> \n> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n>>\n>> These tests check if controls (only exposure time and analogue gain at\n>> the moment) get applied on the frame they were requested for.\n>>\n>> This is tested by looking at the metadata and additionally by\n>> calculating a mean brightness on a centered rect of 20x20 pixels.\n>>\n>> Until today, these tests where only run on a project specific branch\n>> with a modified simple pipeline. In theory they should pass on a\n>> current master :-)\n>>\n>> Current test setup: imx219 with simple pipeline on an imx8mp.\n>> Modifications of either the exposure delay or the gain delay in\n>> the camera_sensor class resulted in test failures.\n>> Which is exactly what this test shall proove.\n>>\n>> Signed-off-by: Stefan Klug \n>> ---\n>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n>> src/apps/lc-compliance/meson.build | 2 +\n>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n>> src/apps/lc-compliance/simple_capture.h | 2 +-\n>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n>> 8 files changed, 487 insertions(+), 3 deletions(-)\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>>\n>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n>> index 1dcfcf92..43fe59f3 100644\n>> --- a/src/apps/lc-compliance/capture_test.cpp\n>> +++ b/src/apps/lc-compliance/capture_test.cpp\n>> @@ -11,6 +11,7 @@\n>> #include \n>>\n>> #include \"environment.h\"\n>> +#include \"per_frame_controls.h\"\n>> #include \"simple_capture.h\"\n>>\n>> using namespace libcamera;\n>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n>> testing::Combine(testing::ValuesIn(ROLES),\n>> testing::ValuesIn(NUMREQUESTS)),\n>> SingleStream::nameParameters);\n>> +\n>> +/*\n>> + * Test Per frame controls\n>> + */\n>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseExposureChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testFramePreciseGainChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseGainChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainIsAppliedOnFirstFrame();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstRequestGetsApplied();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +}\n>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n>> index c792f072..2a6f52af 100644\n>> --- a/src/apps/lc-compliance/meson.build\n>> +++ b/src/apps/lc-compliance/meson.build\n>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n>> 'capture_test.cpp',\n>> 'environment.cpp',\n>> 'main.cpp',\n>> + 'per_frame_controls.cpp',\n>> 'simple_capture.cpp',\n>> + 'time_sheet.cpp',\n>> ])\n>>\n>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n>> new file mode 100644\n>> index 00000000..70fc44ac\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n>> @@ -0,0 +1,214 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.cpp - Tests for per frame controls\n>> + */\n>> +#include \"per_frame_controls.h\"\n>> +\n>> +#include \n>> +\n>> +#include \"time_sheet.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n>> + : SimpleCapture(camera)\n>> +{\n>> +}\n>> +\n>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n>> +{\n>> + ControlList ctrls(camera_->controls().idmap());\n>> + /* Ensure defined default values */\n>> + ctrls.set(controls::AeEnable, false);\n>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n>> + ctrls.set(controls::ExposureTime, 10000);\n>> + ctrls.set(controls::AnalogueGain, 1.0);\n>> +\n>> + if (controls) {\n>> + ctrls.merge(*controls, true);\n>> + }\n>> +\n>> + start(&ctrls);\n>> +\n>> + queueCount_ = 0;\n>> + captureCount_ = 0;\n>> + captureLimit_ = framesToCapture;\n>> +\n>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n>> + timeSheet_ = timeSheet;\n>> + return timeSheet;\n>> +}\n>> +\n>> +int PerFrameControls::queueRequest(Request *request)\n>> +{\n>> + queueCount_++;\n>> + if (queueCount_ > captureLimit_)\n>> + return 0;\n>> +\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->prepareForQueue(request, queueCount_ - 1);\n>> + }\n>> +\n>> + return camera_->queueRequest(request);\n>> +}\n>> +\n>> +void PerFrameControls::requestComplete(Request *request)\n>> +{\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->handleCompleteRequest(request);\n>> + }\n>> +\n>> + captureCount_++;\n>> + if (captureCount_ >= captureLimit_) {\n>> + loop_->exit(0);\n>> + return;\n>> + }\n>> +\n>> + request->reuse(Request::ReuseBuffers);\n>> + if (queueRequest(request))\n>> + loop_->exit(-EINVAL);\n>> +}\n>> +\n>> +void PerFrameControls::runCaptureSession()\n>> +{\n>> + Stream *stream = config_->at(0).stream();\n>> + const std::vector> &buffers = allocator_->buffers(stream);\n>> +\n>> + /* Queue the recommended number of reqeuests. */\n>> + for (const std::unique_ptr &buffer : buffers) {\n>> + std::unique_ptr request = camera_->createRequest();\n>> + request->addBuffer(stream, buffer.get());\n>> + queueRequest(request.get());\n>> + requests_.push_back(std::move(request));\n>> + }\n>> +\n>> + /* Run capture session. */\n>> + loop_ = new EventLoop();\n>> + loop_->exec();\n>> + stop();\n>> + delete loop_;\n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseExposureChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> +\n>> + ts[3].controls().set(controls::ExposureTime, 5000);\n>> + //wait a few frames to settle\n>> + ts[6].controls().set(controls::ExposureTime, 20000);\n>> + ts.printAllInfos();\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n>> +\n>> + /* No increase just before setting exposure */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>> + /*\n>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n>> + * This should be investigated.\n>> + */\n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseGainChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n>> + //wait a few frames to settle\n>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +\n>> + /* No increase just before setting gain */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>> + /*\n>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n>> + */\n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting gain */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 10000);\n>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 8000);\n>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n>> + ts[1].controls().set(controls::ExposureTime, 10000);\n>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n>> +{\n>> + ControlList startValues;\n>> + startValues.set(controls::ExposureTime, 5000);\n>> + startValues.set(controls::AnalogueGain, 1.0);\n>> +\n>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n>> +\n>> + /* Second capture with different values to ensure we don't hit default/old values */\n>> +\n>> + startValues.set(controls::ExposureTime, 15000);\n>> + startValues.set(controls::AnalogueGain, 4.0);\n>> +\n>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n>> +\n>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n>> + EXPECT_GT(brightnessChange, 2.0);\n>> +}\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n>> new file mode 100644\n>> index 00000000..e783f024\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n>> @@ -0,0 +1,41 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.h - Tests for per frame controls\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +\n>> +#include \n>> +\n>> +#include \"../common/event_loop.h\"\n>> +\n>> +#include \"simple_capture.h\"\n>> +#include \"time_sheet.h\"\n>> +\n>> +class PerFrameControls : public SimpleCapture\n>> +{\n>> +public:\n>> + PerFrameControls(std::shared_ptr camera);\n>> +\n>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n>> + void runCaptureSession();\n>> +\n>> + void testFramePreciseExposureChange();\n>> + void testFramePreciseGainChange();\n>> + void testExposureGainIsAppliedOnFirstFrame();\n>> + void testExposureGainFromFirstRequestGetsApplied();\n>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +\n>> + int queueRequest(libcamera::Request *request);\n>> + void requestComplete(libcamera::Request *request) override;\n>> +\n>> + unsigned int queueCount_;\n>> + unsigned int captureCount_;\n>> + unsigned int captureLimit_;\n>> +\n>> + std::weak_ptr timeSheet_;\n>> +};\n>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n>> index cf4d7cf3..56680a83 100644\n>> --- a/src/apps/lc-compliance/simple_capture.cpp\n>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n>> }\n>> }\n>>\n>> -void SimpleCapture::start()\n>> +void SimpleCapture::start(const ControlList *controls)\n>> {\n>> Stream *stream = config_->at(0).stream();\n>> int count = allocator_->allocate(stream);\n>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>>\n>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>>\n>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n>> }\n>>\n>> void SimpleCapture::stop()\n>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n>> index 2911d601..54b1d54b 100644\n>> --- a/src/apps/lc-compliance/simple_capture.h\n>> +++ b/src/apps/lc-compliance/simple_capture.h\n>> @@ -22,7 +22,7 @@ protected:\n>> SimpleCapture(std::shared_ptr camera);\n>> virtual ~SimpleCapture();\n>>\n>> - void start();\n>> + void start(const libcamera::ControlList *controls = nullptr);\n>> void stop();\n>>\n>> virtual void requestComplete(libcamera::Request *request) = 0;\n>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n>> new file mode 100644\n>> index 00000000..9a0e6544\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n>> @@ -0,0 +1,135 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.cpp\n>> + */\n>> +#include \"time_sheet.h\"\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \"libcamera/internal/formats.h\"\n>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +double calcPixelMeanNV12(const uint8_t *data)\n>> +{\n>> + return (double)*data;\n>> +}\n>> +\n>> +double calcPixelMeanRAW10(const uint8_t *data)\n>> +{\n>> + return (double)*((const uint16_t *)data);\n>> +}\n>> +\n>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n>> +{\n>> + const Request::BufferMap &buffers = request->buffers();\n>> + for (const auto &[stream, buffer] : buffers) {\n>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n>> + if (in.isValid()) {\n>> + auto data = in.planes()[0].data();\n>> + auto streamConfig = stream->configuration();\n>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n>> +\n>> + std::function calcPixelMean;\n>> + int pixelStride;\n>> +\n>> + switch (streamConfig.pixelFormat) {\n>> + case formats::NV12:\n>> + calcPixelMean = calcPixelMeanNV12;\n>> + pixelStride = 1;\n>> + break;\n>> + case formats::SRGGB10:\n>> + calcPixelMean = calcPixelMeanRAW10;\n>> + pixelStride = 2;\n>> + break;\n>> + default:\n>> + std::stringstream s;\n>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n>> + throw std::invalid_argument(s.str());\n>> + }\n>> +\n>> + double sum = 0;\n>> + int w = 20;\n>> + int xs = streamConfig.size.width / 2 - w / 2;\n>> + int ys = streamConfig.size.height / 2 - w / 2;\n>> +\n>> + for (auto y = ys; y < ys + w; y++) {\n>> + auto line = data + y * streamConfig.stride;\n>> + for (auto x = xs; x < xs + w; x++) {\n>> + sum += calcPixelMean(line + x * pixelStride);\n>> + }\n>> + }\n>> + sum = sum / (w * w);\n>> + return sum;\n>> + }\n>> + }\n>> + return 0;\n>> +}\n>> +\n>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n>> + : controls_(idmap)\n>> +{\n>> +}\n>> +\n>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n>> +{\n>> + metadata_ = request->metadata();\n>> +\n>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n>> + if (previous) {\n>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n>> + }\n>> + sequence_ = request->sequence();\n>> +}\n>> +\n>> +void TimeSheetEntry::printInfo()\n>> +{\n>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n>> +\n>> + if (!metadata_.empty()) {\n>> + std::cout << \"Metadata:\" << std::endl;\n>> + auto idMap = metadata_.idMap();\n>> + assert(idMap);\n>> + for (const auto &[id, value] : metadata_) {\n>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n>> + }\n>> + }\n>> +}\n>> +\n>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n>> +{\n>> + auto &entry = entries_[pos];\n>> + if (!entry)\n>> + entry = std::make_shared(idmap_);\n>> + return *entry;\n>> +}\n>> +\n>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n>> +{\n>> + request->controls() = get(sequence).controls();\n>> +}\n>> +\n>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n>> +{\n>> + uint32_t sequence = request->sequence();\n>> + auto &entry = get(sequence);\n>> + TimeSheetEntry *previous = nullptr;\n>> + if (sequence >= 1) {\n>> + previous = entries_[sequence - 1].get();\n>> + }\n>> +\n>> + entry.handleCompleteRequest(request, previous);\n>> +}\n>> +\n>> +void TimeSheet::printAllInfos()\n>> +{\n>> + for (auto entry : entries_) {\n>> + if (entry)\n>> + entry->printInfo();\n>> + }\n>> +}\n>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n>> new file mode 100644\n>> index 00000000..c155763c\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.h\n>> @@ -0,0 +1,53 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.h\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +\n>> +class TimeSheetEntry\n>> +{\n>> +public:\n>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n>> +\n>> + libcamera::ControlList &controls() { return controls_; };\n>> + libcamera::ControlList &metadata() { return metadata_; };\n>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n>> + void printInfo();\n>> + double getSpotBrightness() const { return spotBrightness_; };\n>> + double getBrightnessChange() const { return brightnessChange_; };\n>> +\n>> +private:\n>> + double spotBrightness_ = 0.0;\n>> + double brightnessChange_ = 0.0;\n>> + libcamera::ControlList controls_;\n>> + libcamera::ControlList metadata_;\n>> + uint32_t sequence_ = 0;\n>> +};\n>> +\n>> +class TimeSheet\n>> +{\n>> +public:\n>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n>> + : idmap_(idmap), entries_(count){};\n>> +\n>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n>> + void handleCompleteRequest(libcamera::Request *request);\n>> + void printAllInfos();\n>> +\n>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n>> + TimeSheetEntry &get(size_t pos);\n>> +\n>> +private:\n>> + const libcamera::ControlIdMap &idmap_;\n>> + std::vector> entries_;\n>> +};\n>> --\n>> 2.40.1\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id ADA6BBE080\n\tfor ;\n\tFri, 1 Mar 2024 10:48:41 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 01BBD62806;\n\tFri, 1 Mar 2024 11:48:41 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B96BB61C8D\n\tfor ;\n\tFri, 1 Mar 2024 11:48:39 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d] (unknown\n\t[IPv6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 775D9673;\n\tFri, 1 Mar 2024 11:48:25 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"HjCBTljh\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709290105;\n\tbh=QMxqQrNEf0ExMAKan+2a/babizuEu/OU5LQHOh7qpXA=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=HjCBTljh9r9RehWJ8qxaY8oRPNwJDaVtIIxpI3nJOMxITXLPnVWGKGDIFrwNZZBrp\n\tlM/J2G1KrXWlY5vtlWgo3fHcQxe4M9d/7hZAFk30O2IGk3/TkcLBc7c9AzN0p4gIN9\n\tzzfrL6R1oIIWpDmWGz6Lz/1fmZyRfiP09j2sqMfU=","Message-ID":"<1e83d77c-7b98-492b-875f-cda17b87256a@ideasonboard.com>","Date":"Fri, 1 Mar 2024 11:48:36 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"David Plowman ","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t","From":"Stefan Klug ","In-Reply-To":"","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28812,"web_url":"https://patchwork.libcamera.org/comment/28812/","msgid":"","date":"2024-03-01T11:22:12","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi David,\n\nthanks for your comment. This is the same message I sent before, but with proper\nlinewrapping - sorry about that.\n\nAm 01.03.24 um 08:56 schrieb David Plowman:\n> Hi Stefan\n> \n> Thanks for posting this and re-starting the discussion of per-frame controls.\n> \n> Could you perhaps just summarise exactly what you mean by per-frame\n> controls, that is to say, what is being tested here?\n> \n> Eye-balling the code, I think I understood that:\n> \n> * You are setting controls in a specific request.\n> * When that request completes, you are expecting the controls to have\n> taken effect in the images that were returned with that request (but\n> not earlier).\n> \n> But I wasn't totally sure - have I understood that correctly?\n> \n> Also, do we know if any other pipeline handlers implement this\n> behaviour? Do folks think that all pipeline handlers should implement\n> this behaviour?\n> \n> Thanks again!\n> David\n> \nYou are completely right. This needs a bit more context.\n\nIt all started on my side with implementing metadata support in SimplePipeline\nhandler. I soon hit some corner cases where I expected things to behave\ndifferently in cooperation with DelayedControls. So either I hit a bug in\nDelayedControls or I didn't fully understand the concept behind it.\n\nI ended up changing several things in DelayedControls which I believe where\ncorrect. The question then was how to prove correctness as all current users of\nDelayedControls where ISP implementations. There are some unittests, but the\norder of calls in these tests also felt counterintuitive (might well be, that\nit's just missing knowledge on my side).\n\nIn that area there were also no tests in lc-compliance which tested the\nbehaviour of an actual sensor/isp combination. So I started to write these tests\nwith my personal expectation of how I believe things should work. These tests\npass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\nthese tests and hit some corners where work would be required.\n\nBefore digging deeper into that I believe it makes sense to see if my\nexpectations are correct and if a broader audience agrees to them. So here we\nare, that's the reason for the RFC.\n\nNow on to the technical details:\nYes my expectation on per-frame-controls would be: If I queue something for\nframe x, I expect that the system tries to fullfill exacty that request (and\nnever earlier). I'm well aware that this will not be possible in every case when\nan ISP is involved, so the ISP should do the best it can, but the metadata\nshould reflect what was actually achieved. All the tests are currently for\nmanual mode, so no regulation is involved (at least for the params I test).\n\nDo you agree with these assumptions? Looking forward to your opinion.\n\nBest regards,\nStefan\n\n> \n> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n>>\n>> These tests check if controls (only exposure time and analogue gain at\n>> the moment) get applied on the frame they were requested for.\n>>\n>> This is tested by looking at the metadata and additionally by\n>> calculating a mean brightness on a centered rect of 20x20 pixels.\n>>\n>> Until today, these tests where only run on a project specific branch\n>> with a modified simple pipeline. In theory they should pass on a\n>> current master :-)\n>>\n>> Current test setup: imx219 with simple pipeline on an imx8mp.\n>> Modifications of either the exposure delay or the gain delay in\n>> the camera_sensor class resulted in test failures.\n>> Which is exactly what this test shall proove.\n>>\n>> Signed-off-by: Stefan Klug \n>> ---\n>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n>> src/apps/lc-compliance/meson.build | 2 +\n>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n>> src/apps/lc-compliance/simple_capture.h | 2 +-\n>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n>> 8 files changed, 487 insertions(+), 3 deletions(-)\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>>\n>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n>> index 1dcfcf92..43fe59f3 100644\n>> --- a/src/apps/lc-compliance/capture_test.cpp\n>> +++ b/src/apps/lc-compliance/capture_test.cpp\n>> @@ -11,6 +11,7 @@\n>> #include \n>>\n>> #include \"environment.h\"\n>> +#include \"per_frame_controls.h\"\n>> #include \"simple_capture.h\"\n>>\n>> using namespace libcamera;\n>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n>> testing::Combine(testing::ValuesIn(ROLES),\n>> testing::ValuesIn(NUMREQUESTS)),\n>> SingleStream::nameParameters);\n>> +\n>> +/*\n>> + * Test Per frame controls\n>> + */\n>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseExposureChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testFramePreciseGainChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseGainChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainIsAppliedOnFirstFrame();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstRequestGetsApplied();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +}\n>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n>> index c792f072..2a6f52af 100644\n>> --- a/src/apps/lc-compliance/meson.build\n>> +++ b/src/apps/lc-compliance/meson.build\n>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n>> 'capture_test.cpp',\n>> 'environment.cpp',\n>> 'main.cpp',\n>> + 'per_frame_controls.cpp',\n>> 'simple_capture.cpp',\n>> + 'time_sheet.cpp',\n>> ])\n>>\n>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n>> new file mode 100644\n>> index 00000000..70fc44ac\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n>> @@ -0,0 +1,214 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.cpp - Tests for per frame controls\n>> + */\n>> +#include \"per_frame_controls.h\"\n>> +\n>> +#include \n>> +\n>> +#include \"time_sheet.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n>> + : SimpleCapture(camera)\n>> +{\n>> +}\n>> +\n>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n>> +{\n>> + ControlList ctrls(camera_->controls().idmap());\n>> + /* Ensure defined default values */\n>> + ctrls.set(controls::AeEnable, false);\n>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n>> + ctrls.set(controls::ExposureTime, 10000);\n>> + ctrls.set(controls::AnalogueGain, 1.0);\n>> +\n>> + if (controls) {\n>> + ctrls.merge(*controls, true);\n>> + }\n>> +\n>> + start(&ctrls);\n>> +\n>> + queueCount_ = 0;\n>> + captureCount_ = 0;\n>> + captureLimit_ = framesToCapture;\n>> +\n>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n>> + timeSheet_ = timeSheet;\n>> + return timeSheet;\n>> +}\n>> +\n>> +int PerFrameControls::queueRequest(Request *request)\n>> +{\n>> + queueCount_++;\n>> + if (queueCount_ > captureLimit_)\n>> + return 0;\n>> +\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->prepareForQueue(request, queueCount_ - 1);\n>> + }\n>> +\n>> + return camera_->queueRequest(request);\n>> +}\n>> +\n>> +void PerFrameControls::requestComplete(Request *request)\n>> +{\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->handleCompleteRequest(request);\n>> + }\n>> +\n>> + captureCount_++;\n>> + if (captureCount_ >= captureLimit_) {\n>> + loop_->exit(0);\n>> + return;\n>> + }\n>> +\n>> + request->reuse(Request::ReuseBuffers);\n>> + if (queueRequest(request))\n>> + loop_->exit(-EINVAL);\n>> +}\n>> +\n>> +void PerFrameControls::runCaptureSession()\n>> +{\n>> + Stream *stream = config_->at(0).stream();\n>> + const std::vector> &buffers = allocator_->buffers(stream);\n>> +\n>> + /* Queue the recommended number of reqeuests. */\n>> + for (const std::unique_ptr &buffer : buffers) {\n>> + std::unique_ptr request = camera_->createRequest();\n>> + request->addBuffer(stream, buffer.get());\n>> + queueRequest(request.get());\n>> + requests_.push_back(std::move(request));\n>> + }\n>> +\n>> + /* Run capture session. */\n>> + loop_ = new EventLoop();\n>> + loop_->exec();\n>> + stop();\n>> + delete loop_;\n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseExposureChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> +\n>> + ts[3].controls().set(controls::ExposureTime, 5000);\n>> + //wait a few frames to settle\n>> + ts[6].controls().set(controls::ExposureTime, 20000);\n>> + ts.printAllInfos();\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n>> +\n>> + /* No increase just before setting exposure */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>> + /*\n>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n>> + * This should be investigated.\n>> + */\n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseGainChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n>> + //wait a few frames to settle\n>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +\n>> + /* No increase just before setting gain */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>> + /*\n>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n>> + */\n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting gain */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 10000);\n>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 8000);\n>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n>> + ts[1].controls().set(controls::ExposureTime, 10000);\n>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n>> +{\n>> + ControlList startValues;\n>> + startValues.set(controls::ExposureTime, 5000);\n>> + startValues.set(controls::AnalogueGain, 1.0);\n>> +\n>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n>> +\n>> + /* Second capture with different values to ensure we don't hit default/old values */\n>> +\n>> + startValues.set(controls::ExposureTime, 15000);\n>> + startValues.set(controls::AnalogueGain, 4.0);\n>> +\n>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n>> +\n>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n>> + EXPECT_GT(brightnessChange, 2.0);\n>> +}\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n>> new file mode 100644\n>> index 00000000..e783f024\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n>> @@ -0,0 +1,41 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.h - Tests for per frame controls\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +\n>> +#include \n>> +\n>> +#include \"../common/event_loop.h\"\n>> +\n>> +#include \"simple_capture.h\"\n>> +#include \"time_sheet.h\"\n>> +\n>> +class PerFrameControls : public SimpleCapture\n>> +{\n>> +public:\n>> + PerFrameControls(std::shared_ptr camera);\n>> +\n>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n>> + void runCaptureSession();\n>> +\n>> + void testFramePreciseExposureChange();\n>> + void testFramePreciseGainChange();\n>> + void testExposureGainIsAppliedOnFirstFrame();\n>> + void testExposureGainFromFirstRequestGetsApplied();\n>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +\n>> + int queueRequest(libcamera::Request *request);\n>> + void requestComplete(libcamera::Request *request) override;\n>> +\n>> + unsigned int queueCount_;\n>> + unsigned int captureCount_;\n>> + unsigned int captureLimit_;\n>> +\n>> + std::weak_ptr timeSheet_;\n>> +};\n>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n>> index cf4d7cf3..56680a83 100644\n>> --- a/src/apps/lc-compliance/simple_capture.cpp\n>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n>> }\n>> }\n>>\n>> -void SimpleCapture::start()\n>> +void SimpleCapture::start(const ControlList *controls)\n>> {\n>> Stream *stream = config_->at(0).stream();\n>> int count = allocator_->allocate(stream);\n>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>>\n>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>>\n>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n>> }\n>>\n>> void SimpleCapture::stop()\n>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n>> index 2911d601..54b1d54b 100644\n>> --- a/src/apps/lc-compliance/simple_capture.h\n>> +++ b/src/apps/lc-compliance/simple_capture.h\n>> @@ -22,7 +22,7 @@ protected:\n>> SimpleCapture(std::shared_ptr camera);\n>> virtual ~SimpleCapture();\n>>\n>> - void start();\n>> + void start(const libcamera::ControlList *controls = nullptr);\n>> void stop();\n>>\n>> virtual void requestComplete(libcamera::Request *request) = 0;\n>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n>> new file mode 100644\n>> index 00000000..9a0e6544\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n>> @@ -0,0 +1,135 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.cpp\n>> + */\n>> +#include \"time_sheet.h\"\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \"libcamera/internal/formats.h\"\n>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +double calcPixelMeanNV12(const uint8_t *data)\n>> +{\n>> + return (double)*data;\n>> +}\n>> +\n>> +double calcPixelMeanRAW10(const uint8_t *data)\n>> +{\n>> + return (double)*((const uint16_t *)data);\n>> +}\n>> +\n>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n>> +{\n>> + const Request::BufferMap &buffers = request->buffers();\n>> + for (const auto &[stream, buffer] : buffers) {\n>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n>> + if (in.isValid()) {\n>> + auto data = in.planes()[0].data();\n>> + auto streamConfig = stream->configuration();\n>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n>> +\n>> + std::function calcPixelMean;\n>> + int pixelStride;\n>> +\n>> + switch (streamConfig.pixelFormat) {\n>> + case formats::NV12:\n>> + calcPixelMean = calcPixelMeanNV12;\n>> + pixelStride = 1;\n>> + break;\n>> + case formats::SRGGB10:\n>> + calcPixelMean = calcPixelMeanRAW10;\n>> + pixelStride = 2;\n>> + break;\n>> + default:\n>> + std::stringstream s;\n>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n>> + throw std::invalid_argument(s.str());\n>> + }\n>> +\n>> + double sum = 0;\n>> + int w = 20;\n>> + int xs = streamConfig.size.width / 2 - w / 2;\n>> + int ys = streamConfig.size.height / 2 - w / 2;\n>> +\n>> + for (auto y = ys; y < ys + w; y++) {\n>> + auto line = data + y * streamConfig.stride;\n>> + for (auto x = xs; x < xs + w; x++) {\n>> + sum += calcPixelMean(line + x * pixelStride);\n>> + }\n>> + }\n>> + sum = sum / (w * w);\n>> + return sum;\n>> + }\n>> + }\n>> + return 0;\n>> +}\n>> +\n>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n>> + : controls_(idmap)\n>> +{\n>> +}\n>> +\n>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n>> +{\n>> + metadata_ = request->metadata();\n>> +\n>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n>> + if (previous) {\n>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n>> + }\n>> + sequence_ = request->sequence();\n>> +}\n>> +\n>> +void TimeSheetEntry::printInfo()\n>> +{\n>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n>> +\n>> + if (!metadata_.empty()) {\n>> + std::cout << \"Metadata:\" << std::endl;\n>> + auto idMap = metadata_.idMap();\n>> + assert(idMap);\n>> + for (const auto &[id, value] : metadata_) {\n>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n>> + }\n>> + }\n>> +}\n>> +\n>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n>> +{\n>> + auto &entry = entries_[pos];\n>> + if (!entry)\n>> + entry = std::make_shared(idmap_);\n>> + return *entry;\n>> +}\n>> +\n>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n>> +{\n>> + request->controls() = get(sequence).controls();\n>> +}\n>> +\n>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n>> +{\n>> + uint32_t sequence = request->sequence();\n>> + auto &entry = get(sequence);\n>> + TimeSheetEntry *previous = nullptr;\n>> + if (sequence >= 1) {\n>> + previous = entries_[sequence - 1].get();\n>> + }\n>> +\n>> + entry.handleCompleteRequest(request, previous);\n>> +}\n>> +\n>> +void TimeSheet::printAllInfos()\n>> +{\n>> + for (auto entry : entries_) {\n>> + if (entry)\n>> + entry->printInfo();\n>> + }\n>> +}\n>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n>> new file mode 100644\n>> index 00000000..c155763c\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.h\n>> @@ -0,0 +1,53 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.h\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +\n>> +class TimeSheetEntry\n>> +{\n>> +public:\n>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n>> +\n>> + libcamera::ControlList &controls() { return controls_; };\n>> + libcamera::ControlList &metadata() { return metadata_; };\n>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n>> + void printInfo();\n>> + double getSpotBrightness() const { return spotBrightness_; };\n>> + double getBrightnessChange() const { return brightnessChange_; };\n>> +\n>> +private:\n>> + double spotBrightness_ = 0.0;\n>> + double brightnessChange_ = 0.0;\n>> + libcamera::ControlList controls_;\n>> + libcamera::ControlList metadata_;\n>> + uint32_t sequence_ = 0;\n>> +};\n>> +\n>> +class TimeSheet\n>> +{\n>> +public:\n>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n>> + : idmap_(idmap), entries_(count){};\n>> +\n>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n>> + void handleCompleteRequest(libcamera::Request *request);\n>> + void printAllInfos();\n>> +\n>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n>> + TimeSheetEntry &get(size_t pos);\n>> +\n>> +private:\n>> + const libcamera::ControlIdMap &idmap_;\n>> + std::vector> entries_;\n>> +};\n>> --\n>> 2.40.1\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 11919BD160\n\tfor ;\n\tFri, 1 Mar 2024 11:22:18 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 865EA62865;\n\tFri, 1 Mar 2024 12:22:17 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D61126285F\n\tfor ;\n\tFri, 1 Mar 2024 12:22:15 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d] (unknown\n\t[IPv6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8CD769CE;\n\tFri, 1 Mar 2024 12:22:01 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"b7boot5J\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709292121;\n\tbh=fGgoMdoS9Ry3MwgDKQciqpDCV8qyk5ZCIBBUrJGN4X4=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=b7boot5JLYOz+RKBViN68zaOj2+4QdvrxKmSkEkig+YosxgwgCQeBsL8mlugegaD9\n\txwioQPEQthwOvJtSiYAbNFNs6Y1c+vJqIvRYD+wiEUcIujlXt/QKdqhstVhiiZOINq\n\t5PxWE68tcHZvwBzjeNrMT6R1ITS6W80VfXXBsDmk=","Message-ID":"","Date":"Fri, 1 Mar 2024 12:22:12 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"David Plowman ","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t","From":"Stefan Klug ","In-Reply-To":"","Content-Type":"text/plain; charset=UTF-8","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28813,"web_url":"https://patchwork.libcamera.org/comment/28813/","msgid":"","date":"2024-03-01T12:04:41","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Stefan\n\nOn Fri, 1 Mar 2024 at 11:22, Stefan Klug wrote:\n>\n> Hi David,\n>\n> thanks for your comment. This is the same message I sent before, but with proper\n> linewrapping - sorry about that.\n>\n> Am 01.03.24 um 08:56 schrieb David Plowman:\n> > Hi Stefan\n> >\n> > Thanks for posting this and re-starting the discussion of per-frame controls.\n> >\n> > Could you perhaps just summarise exactly what you mean by per-frame\n> > controls, that is to say, what is being tested here?\n> >\n> > Eye-balling the code, I think I understood that:\n> >\n> > * You are setting controls in a specific request.\n> > * When that request completes, you are expecting the controls to have\n> > taken effect in the images that were returned with that request (but\n> > not earlier).\n> >\n> > But I wasn't totally sure - have I understood that correctly?\n> >\n> > Also, do we know if any other pipeline handlers implement this\n> > behaviour? Do folks think that all pipeline handlers should implement\n> > this behaviour?\n> >\n> > Thanks again!\n> > David\n> >\n> You are completely right. This needs a bit more context.\n>\n> It all started on my side with implementing metadata support in SimplePipeline\n> handler. I soon hit some corner cases where I expected things to behave\n> differently in cooperation with DelayedControls. So either I hit a bug in\n> DelayedControls or I didn't fully understand the concept behind it.\n>\n> I ended up changing several things in DelayedControls which I believe where\n> correct. The question then was how to prove correctness as all current users of\n> DelayedControls where ISP implementations. There are some unittests, but the\n> order of calls in these tests also felt counterintuitive (might well be, that\n> it's just missing knowledge on my side).\n>\n> In that area there were also no tests in lc-compliance which tested the\n> behaviour of an actual sensor/isp combination. So I started to write these tests\n> with my personal expectation of how I believe things should work. These tests\n> pass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\n> these tests and hit some corners where work would be required.\n>\n> Before digging deeper into that I believe it makes sense to see if my\n> expectations are correct and if a broader audience agrees to them. So here we\n> are, that's the reason for the RFC.\n>\n> Now on to the technical details:\n> Yes my expectation on per-frame-controls would be: If I queue something for\n> frame x, I expect that the system tries to fullfill exacty that request (and\n> never earlier). I'm well aware that this will not be possible in every case when\n> an ISP is involved, so the ISP should do the best it can, but the metadata\n> should reflect what was actually achieved. All the tests are currently for\n> manual mode, so no regulation is involved (at least for the params I test).\n>\n> Do you agree with these assumptions? Looking forward to your opinion.\n\nThanks for the clarification, and glad that I've understood!\n\nA bit of context from our side... We looked at and implemented a\nper-frame controls solution for the Raspberry Pi back in summer 2022.\nI would say there were a couple of differences:\n\n1. We included a mechanism for reporting failure cases. The only real\nfailure case we have is failing to update camera exposure/gain\nsettings in time because there is quite a short window in which to\nservice the camera frame interrupts (especially when the system is\nbusy and the framerate is fast).\n\nThe mechanism allowed the application to check whether controls had\nbeen applied as expected, or whether they'd been delayed to the next\nframe. It told you whether they'd got \"merged\" with the next set of\ncontrols, or whether all the controls are now running \"a frame late\".\n\n2. We implemented our scheme in the same way as you've described,\nwhere the request where you set the controls has the first images that\nfulfill the request. We called this \"Android mode\" though tbh I'm not\nsure whether or not this really is what Android does!!!\n\nAlthough in the end we decided we didn't like this behaviour so much\nbecause controls get delayed to the back of all the requests that have\nbeen queued (and we normally queue quite a lot so as to avoid the risk\nof frame drops). So we also added \"Raspberry Pi mode\" where controls\nare applied earlier.\n\nIn principle I would be happy to let the Pi run in either mode\n(\"Android\" or \"Pi\"), so long as we can choose \"Pi mode\" for our\napplications. In both cases the same reporting mechanism told the\napplication exactly when the controls had been applied, so that\napplications could be \"mode\" agnostic.\n\nSince Summer 2022, per-frame controls have not really progressed so\nfar as I know. We are intending to resurrect our per-frame controls\nimplementation again once we've got over the Pi 5 libcamera\nintegration, so I'm very happy to be resuming this discussion.\n\nThanks!\nDavid\n\n>\n> Best regards,\n> Stefan\n>\n> >\n> > On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n> >>\n> >> These tests check if controls (only exposure time and analogue gain at\n> >> the moment) get applied on the frame they were requested for.\n> >>\n> >> This is tested by looking at the metadata and additionally by\n> >> calculating a mean brightness on a centered rect of 20x20 pixels.\n> >>\n> >> Until today, these tests where only run on a project specific branch\n> >> with a modified simple pipeline. In theory they should pass on a\n> >> current master :-)\n> >>\n> >> Current test setup: imx219 with simple pipeline on an imx8mp.\n> >> Modifications of either the exposure delay or the gain delay in\n> >> the camera_sensor class resulted in test failures.\n> >> Which is exactly what this test shall proove.\n> >>\n> >> Signed-off-by: Stefan Klug \n> >> ---\n> >> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> >> src/apps/lc-compliance/meson.build | 2 +\n> >> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> >> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> >> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> >> src/apps/lc-compliance/simple_capture.h | 2 +-\n> >> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> >> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> >> 8 files changed, 487 insertions(+), 3 deletions(-)\n> >> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> >> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> >> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> >> create mode 100644 src/apps/lc-compliance/time_sheet.h\n> >>\n> >> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> >> index 1dcfcf92..43fe59f3 100644\n> >> --- a/src/apps/lc-compliance/capture_test.cpp\n> >> +++ b/src/apps/lc-compliance/capture_test.cpp\n> >> @@ -11,6 +11,7 @@\n> >> #include \n> >>\n> >> #include \"environment.h\"\n> >> +#include \"per_frame_controls.h\"\n> >> #include \"simple_capture.h\"\n> >>\n> >> using namespace libcamera;\n> >> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> >> testing::Combine(testing::ValuesIn(ROLES),\n> >> testing::ValuesIn(NUMREQUESTS)),\n> >> SingleStream::nameParameters);\n> >> +\n> >> +/*\n> >> + * Test Per frame controls\n> >> + */\n> >> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> >> +{\n> >> + PerFrameControls capture(camera_);\n> >> + capture.configure(StreamRole::Viewfinder);\n> >> + capture.testFramePreciseExposureChange();\n> >> +}\n> >> +\n> >> +TEST_F(SingleStream, testFramePreciseGainChange)\n> >> +{\n> >> + PerFrameControls capture(camera_);\n> >> + capture.configure(StreamRole::Viewfinder);\n> >> + capture.testFramePreciseGainChange();\n> >> +}\n> >> +\n> >> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> >> +{\n> >> + PerFrameControls capture(camera_);\n> >> + capture.configure(StreamRole::Viewfinder);\n> >> + capture.testExposureGainIsAppliedOnFirstFrame();\n> >> +}\n> >> +\n> >> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> >> +{\n> >> + PerFrameControls capture(camera_);\n> >> + capture.configure(StreamRole::Viewfinder);\n> >> + capture.testExposureGainFromFirstRequestGetsApplied();\n> >> +}\n> >> +\n> >> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> >> +{\n> >> + PerFrameControls capture(camera_);\n> >> + capture.configure(StreamRole::Viewfinder);\n> >> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> >> +}\n> >> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> >> index c792f072..2a6f52af 100644\n> >> --- a/src/apps/lc-compliance/meson.build\n> >> +++ b/src/apps/lc-compliance/meson.build\n> >> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> >> 'capture_test.cpp',\n> >> 'environment.cpp',\n> >> 'main.cpp',\n> >> + 'per_frame_controls.cpp',\n> >> 'simple_capture.cpp',\n> >> + 'time_sheet.cpp',\n> >> ])\n> >>\n> >> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> >> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> >> new file mode 100644\n> >> index 00000000..70fc44ac\n> >> --- /dev/null\n> >> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> >> @@ -0,0 +1,214 @@\n> >> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> >> +/*\n> >> + * Copyright (C) 2024, Ideas on Board Oy\n> >> + *\n> >> + * per_frame_controls.cpp - Tests for per frame controls\n> >> + */\n> >> +#include \"per_frame_controls.h\"\n> >> +\n> >> +#include \n> >> +\n> >> +#include \"time_sheet.h\"\n> >> +\n> >> +using namespace libcamera;\n> >> +\n> >> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> >> + : SimpleCapture(camera)\n> >> +{\n> >> +}\n> >> +\n> >> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> >> +{\n> >> + ControlList ctrls(camera_->controls().idmap());\n> >> + /* Ensure defined default values */\n> >> + ctrls.set(controls::AeEnable, false);\n> >> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> >> + ctrls.set(controls::ExposureTime, 10000);\n> >> + ctrls.set(controls::AnalogueGain, 1.0);\n> >> +\n> >> + if (controls) {\n> >> + ctrls.merge(*controls, true);\n> >> + }\n> >> +\n> >> + start(&ctrls);\n> >> +\n> >> + queueCount_ = 0;\n> >> + captureCount_ = 0;\n> >> + captureLimit_ = framesToCapture;\n> >> +\n> >> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> >> + timeSheet_ = timeSheet;\n> >> + return timeSheet;\n> >> +}\n> >> +\n> >> +int PerFrameControls::queueRequest(Request *request)\n> >> +{\n> >> + queueCount_++;\n> >> + if (queueCount_ > captureLimit_)\n> >> + return 0;\n> >> +\n> >> + auto ts = timeSheet_.lock();\n> >> + if (ts) {\n> >> + ts->prepareForQueue(request, queueCount_ - 1);\n> >> + }\n> >> +\n> >> + return camera_->queueRequest(request);\n> >> +}\n> >> +\n> >> +void PerFrameControls::requestComplete(Request *request)\n> >> +{\n> >> + auto ts = timeSheet_.lock();\n> >> + if (ts) {\n> >> + ts->handleCompleteRequest(request);\n> >> + }\n> >> +\n> >> + captureCount_++;\n> >> + if (captureCount_ >= captureLimit_) {\n> >> + loop_->exit(0);\n> >> + return;\n> >> + }\n> >> +\n> >> + request->reuse(Request::ReuseBuffers);\n> >> + if (queueRequest(request))\n> >> + loop_->exit(-EINVAL);\n> >> +}\n> >> +\n> >> +void PerFrameControls::runCaptureSession()\n> >> +{\n> >> + Stream *stream = config_->at(0).stream();\n> >> + const std::vector> &buffers = allocator_->buffers(stream);\n> >> +\n> >> + /* Queue the recommended number of reqeuests. */\n> >> + for (const std::unique_ptr &buffer : buffers) {\n> >> + std::unique_ptr request = camera_->createRequest();\n> >> + request->addBuffer(stream, buffer.get());\n> >> + queueRequest(request.get());\n> >> + requests_.push_back(std::move(request));\n> >> + }\n> >> +\n> >> + /* Run capture session. */\n> >> + loop_ = new EventLoop();\n> >> + loop_->exec();\n> >> + stop();\n> >> + delete loop_;\n> >> +}\n> >> +\n> >> +void PerFrameControls::testFramePreciseExposureChange()\n> >> +{\n> >> + auto timeSheet = startCaptureWithTimeSheet(10);\n> >> + auto &ts = *timeSheet;\n> >> +\n> >> +\n> >> + ts[3].controls().set(controls::ExposureTime, 5000);\n> >> + //wait a few frames to settle\n> >> + ts[6].controls().set(controls::ExposureTime, 20000);\n> >> + ts.printAllInfos();\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> >> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> >> +\n> >> + /* No increase just before setting exposure */\n> >> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> >> + /*\n> >> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> >> + * This should be investigated.\n> >> + */\n> >> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> >> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> >> +\n> >> + /* No increase just after setting exposure */\n> >> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> >> +\n> >> + /* No increase just after setting exposure */\n> >> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> >> +}\n> >> +\n> >> +void PerFrameControls::testFramePreciseGainChange()\n> >> +{\n> >> + auto timeSheet = startCaptureWithTimeSheet(10);\n> >> + auto &ts = *timeSheet;\n> >> +\n> >> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n> >> + //wait a few frames to settle\n> >> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> >> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >> +\n> >> + /* No increase just before setting gain */\n> >> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> >> + /*\n> >> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> >> + */\n> >> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> >> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> >> +\n> >> + /* No increase just after setting gain */\n> >> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> >> +}\n> >> +\n> >> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> >> +{\n> >> + auto timeSheet = startCaptureWithTimeSheet(5);\n> >> + auto &ts = *timeSheet;\n> >> +\n> >> + ts[0].controls().set(controls::ExposureTime, 10000);\n> >> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + /* We expect it to be applied after 3 frames, the latest*/\n> >> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> >> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >> +}\n> >> +\n> >> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> >> +{\n> >> + auto timeSheet = startCaptureWithTimeSheet(5);\n> >> + auto &ts = *timeSheet;\n> >> +\n> >> + ts[0].controls().set(controls::ExposureTime, 8000);\n> >> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n> >> + ts[1].controls().set(controls::ExposureTime, 10000);\n> >> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + /* We expect it to be applied after 3 frames, the latest*/\n> >> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> >> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >> +}\n> >> +\n> >> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> >> +{\n> >> + ControlList startValues;\n> >> + startValues.set(controls::ExposureTime, 5000);\n> >> + startValues.set(controls::AnalogueGain, 1.0);\n> >> +\n> >> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> >> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> >> +\n> >> + /* Second capture with different values to ensure we don't hit default/old values */\n> >> +\n> >> + startValues.set(controls::ExposureTime, 15000);\n> >> + startValues.set(controls::AnalogueGain, 4.0);\n> >> +\n> >> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> >> +\n> >> + runCaptureSession();\n> >> +\n> >> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> >> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> >> +\n> >> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> >> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> >> + EXPECT_GT(brightnessChange, 2.0);\n> >> +}\n> >> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> >> new file mode 100644\n> >> index 00000000..e783f024\n> >> --- /dev/null\n> >> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> >> @@ -0,0 +1,41 @@\n> >> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> >> +/*\n> >> + * Copyright (C) 2024, Ideas on Board Oy\n> >> + *\n> >> + * per_frame_controls.h - Tests for per frame controls\n> >> + */\n> >> +\n> >> +#pragma once\n> >> +\n> >> +#include \n> >> +\n> >> +#include \n> >> +\n> >> +#include \"../common/event_loop.h\"\n> >> +\n> >> +#include \"simple_capture.h\"\n> >> +#include \"time_sheet.h\"\n> >> +\n> >> +class PerFrameControls : public SimpleCapture\n> >> +{\n> >> +public:\n> >> + PerFrameControls(std::shared_ptr camera);\n> >> +\n> >> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> >> + void runCaptureSession();\n> >> +\n> >> + void testFramePreciseExposureChange();\n> >> + void testFramePreciseGainChange();\n> >> + void testExposureGainIsAppliedOnFirstFrame();\n> >> + void testExposureGainFromFirstRequestGetsApplied();\n> >> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n> >> +\n> >> + int queueRequest(libcamera::Request *request);\n> >> + void requestComplete(libcamera::Request *request) override;\n> >> +\n> >> + unsigned int queueCount_;\n> >> + unsigned int captureCount_;\n> >> + unsigned int captureLimit_;\n> >> +\n> >> + std::weak_ptr timeSheet_;\n> >> +};\n> >> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> >> index cf4d7cf3..56680a83 100644\n> >> --- a/src/apps/lc-compliance/simple_capture.cpp\n> >> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> >> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> >> }\n> >> }\n> >>\n> >> -void SimpleCapture::start()\n> >> +void SimpleCapture::start(const ControlList *controls)\n> >> {\n> >> Stream *stream = config_->at(0).stream();\n> >> int count = allocator_->allocate(stream);\n> >> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n> >>\n> >> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> >>\n> >> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> >> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> >> }\n> >>\n> >> void SimpleCapture::stop()\n> >> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> >> index 2911d601..54b1d54b 100644\n> >> --- a/src/apps/lc-compliance/simple_capture.h\n> >> +++ b/src/apps/lc-compliance/simple_capture.h\n> >> @@ -22,7 +22,7 @@ protected:\n> >> SimpleCapture(std::shared_ptr camera);\n> >> virtual ~SimpleCapture();\n> >>\n> >> - void start();\n> >> + void start(const libcamera::ControlList *controls = nullptr);\n> >> void stop();\n> >>\n> >> virtual void requestComplete(libcamera::Request *request) = 0;\n> >> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> >> new file mode 100644\n> >> index 00000000..9a0e6544\n> >> --- /dev/null\n> >> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> >> @@ -0,0 +1,135 @@\n> >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> >> +/*\n> >> + * Copyright (C) 2024, Ideas on Board Oy\n> >> + *\n> >> + * time_sheet.cpp\n> >> + */\n> >> +#include \"time_sheet.h\"\n> >> +\n> >> +#include \n> >> +#include \n> >> +\n> >> +#include \"libcamera/internal/formats.h\"\n> >> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> >> +\n> >> +using namespace libcamera;\n> >> +\n> >> +double calcPixelMeanNV12(const uint8_t *data)\n> >> +{\n> >> + return (double)*data;\n> >> +}\n> >> +\n> >> +double calcPixelMeanRAW10(const uint8_t *data)\n> >> +{\n> >> + return (double)*((const uint16_t *)data);\n> >> +}\n> >> +\n> >> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> >> +{\n> >> + const Request::BufferMap &buffers = request->buffers();\n> >> + for (const auto &[stream, buffer] : buffers) {\n> >> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> >> + if (in.isValid()) {\n> >> + auto data = in.planes()[0].data();\n> >> + auto streamConfig = stream->configuration();\n> >> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> >> +\n> >> + std::function calcPixelMean;\n> >> + int pixelStride;\n> >> +\n> >> + switch (streamConfig.pixelFormat) {\n> >> + case formats::NV12:\n> >> + calcPixelMean = calcPixelMeanNV12;\n> >> + pixelStride = 1;\n> >> + break;\n> >> + case formats::SRGGB10:\n> >> + calcPixelMean = calcPixelMeanRAW10;\n> >> + pixelStride = 2;\n> >> + break;\n> >> + default:\n> >> + std::stringstream s;\n> >> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n> >> + throw std::invalid_argument(s.str());\n> >> + }\n> >> +\n> >> + double sum = 0;\n> >> + int w = 20;\n> >> + int xs = streamConfig.size.width / 2 - w / 2;\n> >> + int ys = streamConfig.size.height / 2 - w / 2;\n> >> +\n> >> + for (auto y = ys; y < ys + w; y++) {\n> >> + auto line = data + y * streamConfig.stride;\n> >> + for (auto x = xs; x < xs + w; x++) {\n> >> + sum += calcPixelMean(line + x * pixelStride);\n> >> + }\n> >> + }\n> >> + sum = sum / (w * w);\n> >> + return sum;\n> >> + }\n> >> + }\n> >> + return 0;\n> >> +}\n> >> +\n> >> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> >> + : controls_(idmap)\n> >> +{\n> >> +}\n> >> +\n> >> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> >> +{\n> >> + metadata_ = request->metadata();\n> >> +\n> >> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> >> + if (previous) {\n> >> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> >> + }\n> >> + sequence_ = request->sequence();\n> >> +}\n> >> +\n> >> +void TimeSheetEntry::printInfo()\n> >> +{\n> >> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n> >> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> >> +\n> >> + if (!metadata_.empty()) {\n> >> + std::cout << \"Metadata:\" << std::endl;\n> >> + auto idMap = metadata_.idMap();\n> >> + assert(idMap);\n> >> + for (const auto &[id, value] : metadata_) {\n> >> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> >> + }\n> >> + }\n> >> +}\n> >> +\n> >> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> >> +{\n> >> + auto &entry = entries_[pos];\n> >> + if (!entry)\n> >> + entry = std::make_shared(idmap_);\n> >> + return *entry;\n> >> +}\n> >> +\n> >> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> >> +{\n> >> + request->controls() = get(sequence).controls();\n> >> +}\n> >> +\n> >> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> >> +{\n> >> + uint32_t sequence = request->sequence();\n> >> + auto &entry = get(sequence);\n> >> + TimeSheetEntry *previous = nullptr;\n> >> + if (sequence >= 1) {\n> >> + previous = entries_[sequence - 1].get();\n> >> + }\n> >> +\n> >> + entry.handleCompleteRequest(request, previous);\n> >> +}\n> >> +\n> >> +void TimeSheet::printAllInfos()\n> >> +{\n> >> + for (auto entry : entries_) {\n> >> + if (entry)\n> >> + entry->printInfo();\n> >> + }\n> >> +}\n> >> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> >> new file mode 100644\n> >> index 00000000..c155763c\n> >> --- /dev/null\n> >> +++ b/src/apps/lc-compliance/time_sheet.h\n> >> @@ -0,0 +1,53 @@\n> >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> >> +/*\n> >> + * Copyright (C) 2024, Ideas on Board Oy\n> >> + *\n> >> + * time_sheet.h\n> >> + */\n> >> +\n> >> +#pragma once\n> >> +\n> >> +#include \n> >> +#include \n> >> +\n> >> +#include \n> >> +\n> >> +class TimeSheetEntry\n> >> +{\n> >> +public:\n> >> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> >> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> >> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n> >> +\n> >> + libcamera::ControlList &controls() { return controls_; };\n> >> + libcamera::ControlList &metadata() { return metadata_; };\n> >> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> >> + void printInfo();\n> >> + double getSpotBrightness() const { return spotBrightness_; };\n> >> + double getBrightnessChange() const { return brightnessChange_; };\n> >> +\n> >> +private:\n> >> + double spotBrightness_ = 0.0;\n> >> + double brightnessChange_ = 0.0;\n> >> + libcamera::ControlList controls_;\n> >> + libcamera::ControlList metadata_;\n> >> + uint32_t sequence_ = 0;\n> >> +};\n> >> +\n> >> +class TimeSheet\n> >> +{\n> >> +public:\n> >> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> >> + : idmap_(idmap), entries_(count){};\n> >> +\n> >> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> >> + void handleCompleteRequest(libcamera::Request *request);\n> >> + void printAllInfos();\n> >> +\n> >> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> >> + TimeSheetEntry &get(size_t pos);\n> >> +\n> >> +private:\n> >> + const libcamera::ControlIdMap &idmap_;\n> >> + std::vector> entries_;\n> >> +};\n> >> --\n> >> 2.40.1\n> >>\n>\n> --\n> Regards,\n>\n> Stefan Klug\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id C42C8BE080\n\tfor ;\n\tFri, 1 Mar 2024 12:04:56 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 0779762865;\n\tFri, 1 Mar 2024 13:04:56 +0100 (CET)","from mail-oi1-x236.google.com (mail-oi1-x236.google.com\n\t[IPv6:2607:f8b0:4864:20::236])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 5962E6285F\n\tfor ;\n\tFri, 1 Mar 2024 13:04:53 +0100 (CET)","by mail-oi1-x236.google.com with SMTP id\n\t5614622812f47-3c1c51f2fb1so865379b6e.0\n\tfor ;\n\tFri, 01 Mar 2024 04:04:53 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"WokLzDpf\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1709294692; x=1709899492;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=n8mlh3w8+JLHk5fLzbae7d6Zpn3Ew+sXiul8bON6BSw=;\n\tb=WokLzDpfmphdI+RRfjxh4JUPMxkdXdq2XlaWoB+/bQx1B3CmdNqgq2OReEsd94UYRi\n\tV+QtCOqyj0UcqZ5KS5zU5TN9Z5PN8c+oL2sKJZhortNSqvCg0KhPJn29KGdpSUUohx8o\n\tRUZMMhd8wHnwVXuGDs0QzC3MnLZggeaiduC2Tz/Iqe/w9mfWaxdY83pnQxOYCZj1FYH/\n\tCfz8XLo/GKqqKgZmfiCsZJ7GbvtVFWWN9dtOxO3vRBdUrXA94rojMuJi6jfnEPFRiFRH\n\tEZLI+oxk3LIMQ6kG7pirT7J3CP+ldsnoBlyPPbEKnFSDCezIUTTfQrzuOvJ/m2fBvdF7\n\t98Ig==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1709294692; x=1709899492;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=n8mlh3w8+JLHk5fLzbae7d6Zpn3Ew+sXiul8bON6BSw=;\n\tb=Zl79o0GVa2EqfTJHFXIJUp+6kZHku9obE4Y3mJD4uoJYWONU+1W/KeuBAHmh7y+rnM\n\totFp9mzTn7X6IPL1cz/kZdmBZFISqRQ4nlThM7AGRUtjXFAhGIBDZwof6/5C7sjWGovV\n\tb7A+fsxrPKx0QWBlVbTM0bOwZCb12Nef+JI7QD0rDY8WOixYtZw8zidtSjd/G5jasSID\n\tX+Lic4OMQF7rowQ4wWDQvV3x0nF114/PpDLa/1o8F/om2Q+CpN1jwDf7hngCiFtBPei7\n\tA7hPcVoVMuicQ0RWAiRxgbcUtkRnnS6RiBZryLxL0a39rkGEbZWTSDRNmvDLTtWK4DJZ\n\tQ1lw==","X-Gm-Message-State":"AOJu0YzOmpgl9lFVLkKGwxYSGLbWf3Hw/FEuGQd1IeDrj29nh9OLf0+P\n\tckaq8M3nyywOy5/RvFm6mFvl4Ayth73FTFGdO2tPW7vJmP71DUdhHnhU+Swl15B0rKh5Q84bJ0F\n\tKqPhoRocIw/hs9MglNYNsHb3wHAGcaFLuQiJ4ow==","X-Google-Smtp-Source":"AGHT+IFhiV1FHEIYsODgGEQbrl6p5bW5Xh8m/a9OAL2T7+e9nSX3fozElI6TUZDtmCJUHkLD2vSWU4JyqPTLBUmqrFk=","X-Received":"by 2002:a54:4099:0:b0:3c1:c8bd:3b42 with SMTP id\n\ti25-20020a544099000000b003c1c8bd3b42mr1306475oii.25.1709294691838;\n\tFri, 01 Mar 2024 04:04:51 -0800 (PST)","MIME-Version":"1.0","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t\n\t","In-Reply-To":"","From":"David Plowman ","Date":"Fri, 1 Mar 2024 12:04:41 +0000","Message-ID":"","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","To":"Stefan Klug ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28814,"web_url":"https://patchwork.libcamera.org/comment/28814/","msgid":"<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>","date":"2024-03-01T13:22:43","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi David,\n\nAm 01.03.24 um 13:04 schrieb David Plowman:\n> Hi Stefan\n> \n> On Fri, 1 Mar 2024 at 11:22, Stefan Klug wrote:\n>>\n>> Hi David,\n>>\n>> thanks for your comment. This is the same message I sent before, but with proper\n>> linewrapping - sorry about that.\n>>\n>> Am 01.03.24 um 08:56 schrieb David Plowman:\n>>> Hi Stefan\n>>>\n>>> Thanks for posting this and re-starting the discussion of per-frame controls.\n>>>\n>>> Could you perhaps just summarise exactly what you mean by per-frame\n>>> controls, that is to say, what is being tested here?\n>>>\n>>> Eye-balling the code, I think I understood that:\n>>>\n>>> * You are setting controls in a specific request.\n>>> * When that request completes, you are expecting the controls to have\n>>> taken effect in the images that were returned with that request (but\n>>> not earlier).\n>>>\n>>> But I wasn't totally sure - have I understood that correctly?\n>>>\n>>> Also, do we know if any other pipeline handlers implement this\n>>> behaviour? Do folks think that all pipeline handlers should implement\n>>> this behaviour?\n>>>\n>>> Thanks again!\n>>> David\n>>>\n>> You are completely right. This needs a bit more context.\n>>\n>> It all started on my side with implementing metadata support in SimplePipeline\n>> handler. I soon hit some corner cases where I expected things to behave\n>> differently in cooperation with DelayedControls. So either I hit a bug in\n>> DelayedControls or I didn't fully understand the concept behind it.\n>>\n>> I ended up changing several things in DelayedControls which I believe where\n>> correct. The question then was how to prove correctness as all current users of\n>> DelayedControls where ISP implementations. There are some unittests, but the\n>> order of calls in these tests also felt counterintuitive (might well be, that\n>> it's just missing knowledge on my side).\n>>\n>> In that area there were also no tests in lc-compliance which tested the\n>> behaviour of an actual sensor/isp combination. So I started to write these tests\n>> with my personal expectation of how I believe things should work. These tests\n>> pass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\n>> these tests and hit some corners where work would be required.\n>>\n>> Before digging deeper into that I believe it makes sense to see if my\n>> expectations are correct and if a broader audience agrees to them. So here we\n>> are, that's the reason for the RFC.\n>>\n>> Now on to the technical details:\n>> Yes my expectation on per-frame-controls would be: If I queue something for\n>> frame x, I expect that the system tries to fullfill exacty that request (and\n>> never earlier). I'm well aware that this will not be possible in every case when\n>> an ISP is involved, so the ISP should do the best it can, but the metadata\n>> should reflect what was actually achieved. All the tests are currently for\n>> manual mode, so no regulation is involved (at least for the params I test).\n>>\n>> Do you agree with these assumptions? Looking forward to your opinion.\n> \n> Thanks for the clarification, and glad that I've understood!\n> \n> A bit of context from our side... We looked at and implemented a\n> per-frame controls solution for the Raspberry Pi back in summer 2022.\n> I would say there were a couple of differences:\n> \n> 1. We included a mechanism for reporting failure cases. The only real\n> failure case we have is failing to update camera exposure/gain\n> settings in time because there is quite a short window in which to\n> service the camera frame interrupts (especially when the system is\n> busy and the framerate is fast).\n\nThat's interesting. Could you explain the mechanics you used for that\nfeedback or point me to a patchset? Would be interesting to see which\nfailure cases you modelled. How did you test these?\n> \n> The mechanism allowed the application to check whether controls had\n> been applied as expected, or whether they'd been delayed to the next\n> frame. It told you whether they'd got \"merged\" with the next set of\n> controls, or whether all the controls are now running \"a frame late\".\n> \n> 2. We implemented our scheme in the same way as you've described,\n> where the request where you set the controls has the first images that\n> fulfill the request. We called this \"Android mode\" though tbh I'm not\n> sure whether or not this really is what Android does!!!\n> \n> Although in the end we decided we didn't like this behaviour so much\n> because controls get delayed to the back of all the requests that have\n> been queued (and we normally queue quite a lot so as to avoid the risk\n\nWhy was it necessary to move them to the back of all request? I guess in\na typical usecase most requests would not contain controls (at least not\nfrom user side). So there would be a way to apply them as early in the\nqueue as possible. I guess I'm missing somethimng here.\n\n> of frame drops). So we also added \"Raspberry Pi mode\" where controls\n> are applied earlier.\n> \n> In principle I would be happy to let the Pi run in either mode\n> (\"Android\" or \"Pi\"), so long as we can choose \"Pi mode\" for our\n> applications. In both cases the same reporting mechanism told the\n> application exactly when the controls had been applied, so that\n> applications could be \"mode\" agnostic.\n> \n> Since Summer 2022, per-frame controls have not really progressed so\n> far as I know. We are intending to resurrect our per-frame controls\n> implementation again once we've got over the Pi 5 libcamera\n> integration, so I'm very happy to be resuming this discussion.\n\nSure. Would be great if we could come up with a set of tests that wee\nall agree on.\n\nCheers,\nStefan\n\n> \n> Thanks!\n> David\n> \n>>\n>> Best regards,\n>> Stefan\n>>\n>>>\n>>> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n>>>>\n>>>> These tests check if controls (only exposure time and analogue gain at\n>>>> the moment) get applied on the frame they were requested for.\n>>>>\n>>>> This is tested by looking at the metadata and additionally by\n>>>> calculating a mean brightness on a centered rect of 20x20 pixels.\n>>>>\n>>>> Until today, these tests where only run on a project specific branch\n>>>> with a modified simple pipeline. In theory they should pass on a\n>>>> current master :-)\n>>>>\n>>>> Current test setup: imx219 with simple pipeline on an imx8mp.\n>>>> Modifications of either the exposure delay or the gain delay in\n>>>> the camera_sensor class resulted in test failures.\n>>>> Which is exactly what this test shall proove.\n>>>>\n>>>> Signed-off-by: Stefan Klug \n>>>> ---\n>>>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n>>>> src/apps/lc-compliance/meson.build | 2 +\n>>>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n>>>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n>>>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n>>>> src/apps/lc-compliance/simple_capture.h | 2 +-\n>>>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n>>>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n>>>> 8 files changed, 487 insertions(+), 3 deletions(-)\n>>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n>>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n>>>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n>>>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>>>>\n>>>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n>>>> index 1dcfcf92..43fe59f3 100644\n>>>> --- a/src/apps/lc-compliance/capture_test.cpp\n>>>> +++ b/src/apps/lc-compliance/capture_test.cpp\n>>>> @@ -11,6 +11,7 @@\n>>>> #include \n>>>>\n>>>> #include \"environment.h\"\n>>>> +#include \"per_frame_controls.h\"\n>>>> #include \"simple_capture.h\"\n>>>>\n>>>> using namespace libcamera;\n>>>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n>>>> testing::Combine(testing::ValuesIn(ROLES),\n>>>> testing::ValuesIn(NUMREQUESTS)),\n>>>> SingleStream::nameParameters);\n>>>> +\n>>>> +/*\n>>>> + * Test Per frame controls\n>>>> + */\n>>>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n>>>> +{\n>>>> + PerFrameControls capture(camera_);\n>>>> + capture.configure(StreamRole::Viewfinder);\n>>>> + capture.testFramePreciseExposureChange();\n>>>> +}\n>>>> +\n>>>> +TEST_F(SingleStream, testFramePreciseGainChange)\n>>>> +{\n>>>> + PerFrameControls capture(camera_);\n>>>> + capture.configure(StreamRole::Viewfinder);\n>>>> + capture.testFramePreciseGainChange();\n>>>> +}\n>>>> +\n>>>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n>>>> +{\n>>>> + PerFrameControls capture(camera_);\n>>>> + capture.configure(StreamRole::Viewfinder);\n>>>> + capture.testExposureGainIsAppliedOnFirstFrame();\n>>>> +}\n>>>> +\n>>>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n>>>> +{\n>>>> + PerFrameControls capture(camera_);\n>>>> + capture.configure(StreamRole::Viewfinder);\n>>>> + capture.testExposureGainFromFirstRequestGetsApplied();\n>>>> +}\n>>>> +\n>>>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n>>>> +{\n>>>> + PerFrameControls capture(camera_);\n>>>> + capture.configure(StreamRole::Viewfinder);\n>>>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n>>>> +}\n>>>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n>>>> index c792f072..2a6f52af 100644\n>>>> --- a/src/apps/lc-compliance/meson.build\n>>>> +++ b/src/apps/lc-compliance/meson.build\n>>>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n>>>> 'capture_test.cpp',\n>>>> 'environment.cpp',\n>>>> 'main.cpp',\n>>>> + 'per_frame_controls.cpp',\n>>>> 'simple_capture.cpp',\n>>>> + 'time_sheet.cpp',\n>>>> ])\n>>>>\n>>>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n>>>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n>>>> new file mode 100644\n>>>> index 00000000..70fc44ac\n>>>> --- /dev/null\n>>>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n>>>> @@ -0,0 +1,214 @@\n>>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>>>> +/*\n>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>> + *\n>>>> + * per_frame_controls.cpp - Tests for per frame controls\n>>>> + */\n>>>> +#include \"per_frame_controls.h\"\n>>>> +\n>>>> +#include \n>>>> +\n>>>> +#include \"time_sheet.h\"\n>>>> +\n>>>> +using namespace libcamera;\n>>>> +\n>>>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n>>>> + : SimpleCapture(camera)\n>>>> +{\n>>>> +}\n>>>> +\n>>>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n>>>> +{\n>>>> + ControlList ctrls(camera_->controls().idmap());\n>>>> + /* Ensure defined default values */\n>>>> + ctrls.set(controls::AeEnable, false);\n>>>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n>>>> + ctrls.set(controls::ExposureTime, 10000);\n>>>> + ctrls.set(controls::AnalogueGain, 1.0);\n>>>> +\n>>>> + if (controls) {\n>>>> + ctrls.merge(*controls, true);\n>>>> + }\n>>>> +\n>>>> + start(&ctrls);\n>>>> +\n>>>> + queueCount_ = 0;\n>>>> + captureCount_ = 0;\n>>>> + captureLimit_ = framesToCapture;\n>>>> +\n>>>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n>>>> + timeSheet_ = timeSheet;\n>>>> + return timeSheet;\n>>>> +}\n>>>> +\n>>>> +int PerFrameControls::queueRequest(Request *request)\n>>>> +{\n>>>> + queueCount_++;\n>>>> + if (queueCount_ > captureLimit_)\n>>>> + return 0;\n>>>> +\n>>>> + auto ts = timeSheet_.lock();\n>>>> + if (ts) {\n>>>> + ts->prepareForQueue(request, queueCount_ - 1);\n>>>> + }\n>>>> +\n>>>> + return camera_->queueRequest(request);\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::requestComplete(Request *request)\n>>>> +{\n>>>> + auto ts = timeSheet_.lock();\n>>>> + if (ts) {\n>>>> + ts->handleCompleteRequest(request);\n>>>> + }\n>>>> +\n>>>> + captureCount_++;\n>>>> + if (captureCount_ >= captureLimit_) {\n>>>> + loop_->exit(0);\n>>>> + return;\n>>>> + }\n>>>> +\n>>>> + request->reuse(Request::ReuseBuffers);\n>>>> + if (queueRequest(request))\n>>>> + loop_->exit(-EINVAL);\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::runCaptureSession()\n>>>> +{\n>>>> + Stream *stream = config_->at(0).stream();\n>>>> + const std::vector> &buffers = allocator_->buffers(stream);\n>>>> +\n>>>> + /* Queue the recommended number of reqeuests. */\n>>>> + for (const std::unique_ptr &buffer : buffers) {\n>>>> + std::unique_ptr request = camera_->createRequest();\n>>>> + request->addBuffer(stream, buffer.get());\n>>>> + queueRequest(request.get());\n>>>> + requests_.push_back(std::move(request));\n>>>> + }\n>>>> +\n>>>> + /* Run capture session. */\n>>>> + loop_ = new EventLoop();\n>>>> + loop_->exec();\n>>>> + stop();\n>>>> + delete loop_;\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::testFramePreciseExposureChange()\n>>>> +{\n>>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>>>> + auto &ts = *timeSheet;\n>>>> +\n>>>> +\n>>>> + ts[3].controls().set(controls::ExposureTime, 5000);\n>>>> + //wait a few frames to settle\n>>>> + ts[6].controls().set(controls::ExposureTime, 20000);\n>>>> + ts.printAllInfos();\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>>>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n>>>> +\n>>>> + /* No increase just before setting exposure */\n>>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>>>> + /*\n>>>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n>>>> + * This should be investigated.\n>>>> + */\n>>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>>>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>>>> +\n>>>> + /* No increase just after setting exposure */\n>>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>>>> +\n>>>> + /* No increase just after setting exposure */\n>>>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::testFramePreciseGainChange()\n>>>> +{\n>>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>>>> + auto &ts = *timeSheet;\n>>>> +\n>>>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n>>>> + //wait a few frames to settle\n>>>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n>>>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>> +\n>>>> + /* No increase just before setting gain */\n>>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>>>> + /*\n>>>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n>>>> + */\n>>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>>>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>>>> +\n>>>> + /* No increase just after setting gain */\n>>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n>>>> +{\n>>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>>>> + auto &ts = *timeSheet;\n>>>> +\n>>>> + ts[0].controls().set(controls::ExposureTime, 10000);\n>>>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + /* We expect it to be applied after 3 frames, the latest*/\n>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n>>>> +{\n>>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>>>> + auto &ts = *timeSheet;\n>>>> +\n>>>> + ts[0].controls().set(controls::ExposureTime, 8000);\n>>>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n>>>> + ts[1].controls().set(controls::ExposureTime, 10000);\n>>>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + /* We expect it to be applied after 3 frames, the latest*/\n>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>> +}\n>>>> +\n>>>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n>>>> +{\n>>>> + ControlList startValues;\n>>>> + startValues.set(controls::ExposureTime, 5000);\n>>>> + startValues.set(controls::AnalogueGain, 1.0);\n>>>> +\n>>>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n>>>> +\n>>>> + /* Second capture with different values to ensure we don't hit default/old values */\n>>>> +\n>>>> + startValues.set(controls::ExposureTime, 15000);\n>>>> + startValues.set(controls::AnalogueGain, 4.0);\n>>>> +\n>>>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n>>>> +\n>>>> + runCaptureSession();\n>>>> +\n>>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n>>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n>>>> +\n>>>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n>>>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n>>>> + EXPECT_GT(brightnessChange, 2.0);\n>>>> +}\n>>>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n>>>> new file mode 100644\n>>>> index 00000000..e783f024\n>>>> --- /dev/null\n>>>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n>>>> @@ -0,0 +1,41 @@\n>>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>>>> +/*\n>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>> + *\n>>>> + * per_frame_controls.h - Tests for per frame controls\n>>>> + */\n>>>> +\n>>>> +#pragma once\n>>>> +\n>>>> +#include \n>>>> +\n>>>> +#include \n>>>> +\n>>>> +#include \"../common/event_loop.h\"\n>>>> +\n>>>> +#include \"simple_capture.h\"\n>>>> +#include \"time_sheet.h\"\n>>>> +\n>>>> +class PerFrameControls : public SimpleCapture\n>>>> +{\n>>>> +public:\n>>>> + PerFrameControls(std::shared_ptr camera);\n>>>> +\n>>>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n>>>> + void runCaptureSession();\n>>>> +\n>>>> + void testFramePreciseExposureChange();\n>>>> + void testFramePreciseGainChange();\n>>>> + void testExposureGainIsAppliedOnFirstFrame();\n>>>> + void testExposureGainFromFirstRequestGetsApplied();\n>>>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n>>>> +\n>>>> + int queueRequest(libcamera::Request *request);\n>>>> + void requestComplete(libcamera::Request *request) override;\n>>>> +\n>>>> + unsigned int queueCount_;\n>>>> + unsigned int captureCount_;\n>>>> + unsigned int captureLimit_;\n>>>> +\n>>>> + std::weak_ptr timeSheet_;\n>>>> +};\n>>>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n>>>> index cf4d7cf3..56680a83 100644\n>>>> --- a/src/apps/lc-compliance/simple_capture.cpp\n>>>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n>>>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n>>>> }\n>>>> }\n>>>>\n>>>> -void SimpleCapture::start()\n>>>> +void SimpleCapture::start(const ControlList *controls)\n>>>> {\n>>>> Stream *stream = config_->at(0).stream();\n>>>> int count = allocator_->allocate(stream);\n>>>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>>>>\n>>>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>>>>\n>>>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n>>>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n>>>> }\n>>>>\n>>>> void SimpleCapture::stop()\n>>>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n>>>> index 2911d601..54b1d54b 100644\n>>>> --- a/src/apps/lc-compliance/simple_capture.h\n>>>> +++ b/src/apps/lc-compliance/simple_capture.h\n>>>> @@ -22,7 +22,7 @@ protected:\n>>>> SimpleCapture(std::shared_ptr camera);\n>>>> virtual ~SimpleCapture();\n>>>>\n>>>> - void start();\n>>>> + void start(const libcamera::ControlList *controls = nullptr);\n>>>> void stop();\n>>>>\n>>>> virtual void requestComplete(libcamera::Request *request) = 0;\n>>>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n>>>> new file mode 100644\n>>>> index 00000000..9a0e6544\n>>>> --- /dev/null\n>>>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n>>>> @@ -0,0 +1,135 @@\n>>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>>>> +/*\n>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>> + *\n>>>> + * time_sheet.cpp\n>>>> + */\n>>>> +#include \"time_sheet.h\"\n>>>> +\n>>>> +#include \n>>>> +#include \n>>>> +\n>>>> +#include \"libcamera/internal/formats.h\"\n>>>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>>>> +\n>>>> +using namespace libcamera;\n>>>> +\n>>>> +double calcPixelMeanNV12(const uint8_t *data)\n>>>> +{\n>>>> + return (double)*data;\n>>>> +}\n>>>> +\n>>>> +double calcPixelMeanRAW10(const uint8_t *data)\n>>>> +{\n>>>> + return (double)*((const uint16_t *)data);\n>>>> +}\n>>>> +\n>>>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n>>>> +{\n>>>> + const Request::BufferMap &buffers = request->buffers();\n>>>> + for (const auto &[stream, buffer] : buffers) {\n>>>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n>>>> + if (in.isValid()) {\n>>>> + auto data = in.planes()[0].data();\n>>>> + auto streamConfig = stream->configuration();\n>>>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n>>>> +\n>>>> + std::function calcPixelMean;\n>>>> + int pixelStride;\n>>>> +\n>>>> + switch (streamConfig.pixelFormat) {\n>>>> + case formats::NV12:\n>>>> + calcPixelMean = calcPixelMeanNV12;\n>>>> + pixelStride = 1;\n>>>> + break;\n>>>> + case formats::SRGGB10:\n>>>> + calcPixelMean = calcPixelMeanRAW10;\n>>>> + pixelStride = 2;\n>>>> + break;\n>>>> + default:\n>>>> + std::stringstream s;\n>>>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n>>>> + throw std::invalid_argument(s.str());\n>>>> + }\n>>>> +\n>>>> + double sum = 0;\n>>>> + int w = 20;\n>>>> + int xs = streamConfig.size.width / 2 - w / 2;\n>>>> + int ys = streamConfig.size.height / 2 - w / 2;\n>>>> +\n>>>> + for (auto y = ys; y < ys + w; y++) {\n>>>> + auto line = data + y * streamConfig.stride;\n>>>> + for (auto x = xs; x < xs + w; x++) {\n>>>> + sum += calcPixelMean(line + x * pixelStride);\n>>>> + }\n>>>> + }\n>>>> + sum = sum / (w * w);\n>>>> + return sum;\n>>>> + }\n>>>> + }\n>>>> + return 0;\n>>>> +}\n>>>> +\n>>>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n>>>> + : controls_(idmap)\n>>>> +{\n>>>> +}\n>>>> +\n>>>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n>>>> +{\n>>>> + metadata_ = request->metadata();\n>>>> +\n>>>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n>>>> + if (previous) {\n>>>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n>>>> + }\n>>>> + sequence_ = request->sequence();\n>>>> +}\n>>>> +\n>>>> +void TimeSheetEntry::printInfo()\n>>>> +{\n>>>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n>>>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n>>>> +\n>>>> + if (!metadata_.empty()) {\n>>>> + std::cout << \"Metadata:\" << std::endl;\n>>>> + auto idMap = metadata_.idMap();\n>>>> + assert(idMap);\n>>>> + for (const auto &[id, value] : metadata_) {\n>>>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n>>>> + }\n>>>> + }\n>>>> +}\n>>>> +\n>>>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n>>>> +{\n>>>> + auto &entry = entries_[pos];\n>>>> + if (!entry)\n>>>> + entry = std::make_shared(idmap_);\n>>>> + return *entry;\n>>>> +}\n>>>> +\n>>>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n>>>> +{\n>>>> + request->controls() = get(sequence).controls();\n>>>> +}\n>>>> +\n>>>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n>>>> +{\n>>>> + uint32_t sequence = request->sequence();\n>>>> + auto &entry = get(sequence);\n>>>> + TimeSheetEntry *previous = nullptr;\n>>>> + if (sequence >= 1) {\n>>>> + previous = entries_[sequence - 1].get();\n>>>> + }\n>>>> +\n>>>> + entry.handleCompleteRequest(request, previous);\n>>>> +}\n>>>> +\n>>>> +void TimeSheet::printAllInfos()\n>>>> +{\n>>>> + for (auto entry : entries_) {\n>>>> + if (entry)\n>>>> + entry->printInfo();\n>>>> + }\n>>>> +}\n>>>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n>>>> new file mode 100644\n>>>> index 00000000..c155763c\n>>>> --- /dev/null\n>>>> +++ b/src/apps/lc-compliance/time_sheet.h\n>>>> @@ -0,0 +1,53 @@\n>>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>>>> +/*\n>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>> + *\n>>>> + * time_sheet.h\n>>>> + */\n>>>> +\n>>>> +#pragma once\n>>>> +\n>>>> +#include \n>>>> +#include \n>>>> +\n>>>> +#include \n>>>> +\n>>>> +class TimeSheetEntry\n>>>> +{\n>>>> +public:\n>>>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n>>>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n>>>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n>>>> +\n>>>> + libcamera::ControlList &controls() { return controls_; };\n>>>> + libcamera::ControlList &metadata() { return metadata_; };\n>>>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n>>>> + void printInfo();\n>>>> + double getSpotBrightness() const { return spotBrightness_; };\n>>>> + double getBrightnessChange() const { return brightnessChange_; };\n>>>> +\n>>>> +private:\n>>>> + double spotBrightness_ = 0.0;\n>>>> + double brightnessChange_ = 0.0;\n>>>> + libcamera::ControlList controls_;\n>>>> + libcamera::ControlList metadata_;\n>>>> + uint32_t sequence_ = 0;\n>>>> +};\n>>>> +\n>>>> +class TimeSheet\n>>>> +{\n>>>> +public:\n>>>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n>>>> + : idmap_(idmap), entries_(count){};\n>>>> +\n>>>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n>>>> + void handleCompleteRequest(libcamera::Request *request);\n>>>> + void printAllInfos();\n>>>> +\n>>>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n>>>> + TimeSheetEntry &get(size_t pos);\n>>>> +\n>>>> +private:\n>>>> + const libcamera::ControlIdMap &idmap_;\n>>>> + std::vector> entries_;\n>>>> +};\n>>>> --\n>>>> 2.40.1\n>>>>\n>>\n>> --\n>> Regards,\n>>\n>> Stefan Klug\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id ED309BD160\n\tfor ;\n\tFri, 1 Mar 2024 13:22:48 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 375DD62865;\n\tFri, 1 Mar 2024 14:22:48 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B45BE6285F\n\tfor ;\n\tFri, 1 Mar 2024 14:22:46 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d] (unknown\n\t[IPv6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 6636D9CE;\n\tFri, 1 Mar 2024 14:22:32 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"kgO8Kv7R\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709299352;\n\tbh=o03ENlhhFD+yHRLpDg/LbydO5p+W3eqMR6Jr4+w+hXg=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=kgO8Kv7RCLOoAb+Ghuw/21X8fBpp0cw2Df29NLHm7IAwgwxaYUlUzsXsuJGVUE/ZE\n\t2/5+YmbaQxPFxNa1xTSb00GolCYXfngpwbwOy4/uJTUUM94YGViWTdUhsAC5Lbo2pA\n\tneERV+HeVxXEgnuiGRHyCnBihJdolqdjCxcwWI2M=","Message-ID":"<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>","Date":"Fri, 1 Mar 2024 14:22:43 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"David Plowman ","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t\n\t\n\t","From":"Stefan Klug ","In-Reply-To":"","Content-Type":"text/plain; charset=UTF-8","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28815,"web_url":"https://patchwork.libcamera.org/comment/28815/","msgid":"","date":"2024-03-01T13:34:52","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi Kieran,\n\nthanks for you review. I will fix the commit/style issues in the first patchset.\nFor now I was more focused on starting a dicussion on the overall idea.\n\nAm 29.02.24 um 18:36 schrieb Kieran Bingham:\n> Hi Stefan,\n> \n> Some quick review notes.\n> \n> I'm happy to see a test that actually tries to validate the\n> DelayedControls implementations, and considers the real effects on the\n> sensor.\n> \n> We try to break patches down to single topic commits, so here I can see\n> at least\n> - The introduction of the TimeSheet helper class\n> - Fix the SimpleCapture::start() to accept a control list\n> - Add per-frame-control tests>\n> Quoting Stefan Klug (2024-02-29 17:01:15)\n>> These tests check if controls (only exposure time and analogue gain at\n>> the moment) get applied on the frame they were requested for.\n>>\n>> This is tested by looking at the metadata and additionally by\n>> calculating a mean brightness on a centered rect of 20x20 pixels.\n>>\n>> Until today, these tests where only run on a project specific branch\n>> with a modified simple pipeline. In theory they should pass on a\n>> current master :-)\n> \n> I love good theories! ;-) I'm curious to see what happens on master and\n> other platforms.\n> \n> \n>> Current test setup: imx219 with simple pipeline on an imx8mp.\n>> Modifications of either the exposure delay or the gain delay in\n>> the camera_sensor class resulted in test failures.\n>> Which is exactly what this test shall proove.\n> \n> 'prove' ;-)\n> \n>>\n>> Signed-off-by: Stefan Klug \n>> ---\n>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n>> src/apps/lc-compliance/meson.build | 2 +\n>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n>> src/apps/lc-compliance/simple_capture.h | 2 +-\n>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n>> 8 files changed, 487 insertions(+), 3 deletions(-)\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>>\n>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n>> index 1dcfcf92..43fe59f3 100644\n>> --- a/src/apps/lc-compliance/capture_test.cpp\n>> +++ b/src/apps/lc-compliance/capture_test.cpp\n>> @@ -11,6 +11,7 @@\n>> #include \n>> \n>> #include \"environment.h\"\n>> +#include \"per_frame_controls.h\"\n>> #include \"simple_capture.h\"\n>> \n>> using namespace libcamera;\n>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n>> testing::Combine(testing::ValuesIn(ROLES),\n>> testing::ValuesIn(NUMREQUESTS)),\n>> SingleStream::nameParameters);\n>> +\n>> +/*\n>> + * Test Per frame controls\n>> + */\n>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseExposureChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testFramePreciseGainChange)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testFramePreciseGainChange();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainIsAppliedOnFirstFrame();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstRequestGetsApplied();\n>> +}\n>> +\n>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n>> +{\n>> + PerFrameControls capture(camera_);\n>> + capture.configure(StreamRole::Viewfinder);\n>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +}\n>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n>> index c792f072..2a6f52af 100644\n>> --- a/src/apps/lc-compliance/meson.build\n>> +++ b/src/apps/lc-compliance/meson.build\n>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n>> 'capture_test.cpp',\n>> 'environment.cpp',\n>> 'main.cpp',\n>> + 'per_frame_controls.cpp',\n>> 'simple_capture.cpp',\n>> + 'time_sheet.cpp',\n>> ])\n>> \n>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n>> new file mode 100644\n>> index 00000000..70fc44ac\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n>> @@ -0,0 +1,214 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.cpp - Tests for per frame controls\n>> + */\n>> +#include \"per_frame_controls.h\"\n>> +\n>> +#include \n>> +\n>> +#include \"time_sheet.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n>> + : SimpleCapture(camera)\n>> +{\n>> +}\n>> +\n>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n>> +{\n>> + ControlList ctrls(camera_->controls().idmap());\n>> + /* Ensure defined default values */\n>> + ctrls.set(controls::AeEnable, false);\n>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n>> + ctrls.set(controls::ExposureTime, 10000);\n>> + ctrls.set(controls::AnalogueGain, 1.0);\n>> +\n>> + if (controls) {\n>> + ctrls.merge(*controls, true);\n>> + }\n> \n> No need for { } on a single line statement.\n> \tif (controls)\n> \t\tctrls.merge(*controls, true);\n> \n> is sufficient in our code style.\n> \n> \n>> +\n>> + start(&ctrls);\n>> +\n>> + queueCount_ = 0;\n>> + captureCount_ = 0;\n>> + captureLimit_ = framesToCapture;\n>> +\n>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n>> + timeSheet_ = timeSheet;\n> \n> I'm curious. Why shared and not make_unique? Is it kept / shared in\n> multiple locations? I guess I'll see later.\n> \n>> + return timeSheet;\n>> +}\n>> +\n>> +int PerFrameControls::queueRequest(Request *request)\n>> +{\n>> + queueCount_++;\n>> + if (queueCount_ > captureLimit_)\n> \n> Should this return an error as it's /not/ queueing? Maybe not essential\n> ... lets see how it's used...\n> \n> \n> Probably not as we just want to stop queueing and we still want to wait\n> for the requests queued to complete...\n> \n> \n>> + return 0;\n>> +\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->prepareForQueue(request, queueCount_ - 1);\n>> + }\n> \n> Single line so no braces, same elsewhere.\n> \n>> +\n>> + return camera_->queueRequest(request);\n>> +}\n>> +\n>> +void PerFrameControls::requestComplete(Request *request)\n>> +{\n>> + auto ts = timeSheet_.lock();\n>> + if (ts) {\n>> + ts->handleCompleteRequest(request);\n>> + }\n>> +\n>> + captureCount_++;\n>> + if (captureCount_ >= captureLimit_) {\n>> + loop_->exit(0);\n>> + return;\n>> + }\n>> +\n>> + request->reuse(Request::ReuseBuffers);\n>> + if (queueRequest(request))\n>> + loop_->exit(-EINVAL);\n>> +}\n>> +\n>> +void PerFrameControls::runCaptureSession()\n>> +{\n>> + Stream *stream = config_->at(0).stream();\n>> + const std::vector> &buffers = allocator_->buffers(stream);\n>> +\n>> + /* Queue the recommended number of reqeuests. */\n>> + for (const std::unique_ptr &buffer : buffers) {\n>> + std::unique_ptr request = camera_->createRequest();\n>> + request->addBuffer(stream, buffer.get());\n>> + queueRequest(request.get());\n>> + requests_.push_back(std::move(request));\n>> + }\n>> +\n>> + /* Run capture session. */\n>> + loop_ = new EventLoop();\n>> + loop_->exec();\n>> + stop();\n>> + delete loop_;\n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseExposureChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> +\n>> + ts[3].controls().set(controls::ExposureTime, 5000);\n>> + //wait a few frames to settle\n> \n> \t/* Single line comment style. */\n> \n>> + ts[6].controls().set(controls::ExposureTime, 20000);\n>> + ts.printAllInfos();\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n>> +\n>> + /* No increase just before setting exposure */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> \n> I'd wrap this a little, but it's better to keep the comment in one.\n> Perhaps:\n> \n> \tEXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05)\n> \t\t<< \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> \n>> + /*\n>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n>> + * This should be investigated.\n>> + */\n> \n> indentation is broken here. Also lines could be wrapped to target\n> 80chars.\n> \n> \n> \n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +\n>> + /* No increase just after setting exposure */\n>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> \n> Same here, I'd wrap the message to the line below. But that's all we can\n> do here I think.\n> \n> \"changed too much2 frames\" probably needs cleaning up.\n> \n>> +}\n>> +\n>> +void PerFrameControls::testFramePreciseGainChange()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n>> + //wait a few frames to settle\n>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +\n>> + /* No increase just before setting gain */\n>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>> + /*\n>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n>> + */\n> \n> Same indentation issues here, and wrapping where possible.\n> \n>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>> +\n>> + /* No increase just after setting gain */\n>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 10000);\n>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n> \n> \"latest */\"\n> \n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n>> +{\n>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>> + auto &ts = *timeSheet;\n>> +\n>> + ts[0].controls().set(controls::ExposureTime, 8000);\n>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n>> + ts[1].controls().set(controls::ExposureTime, 10000);\n>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n>> +\n>> + runCaptureSession();\n>> +\n>> + /* We expect it to be applied after 3 frames, the latest*/\n> \n> \"latest */\"\n> \n>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>> +}\n>> +\n>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n>> +{\n>> + ControlList startValues;\n>> + startValues.set(controls::ExposureTime, 5000);\n>> + startValues.set(controls::AnalogueGain, 1.0);\n>> +\n>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n>> +\n>> + /* Second capture with different values to ensure we don't hit default/old values */\n>> +\n>> + startValues.set(controls::ExposureTime, 15000);\n>> + startValues.set(controls::AnalogueGain, 4.0);\n>> +\n>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n>> +\n>> + runCaptureSession();\n>> +\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n>> +\n>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n>> + EXPECT_GT(brightnessChange, 2.0);\n>> +}\n>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n>> new file mode 100644\n>> index 00000000..e783f024\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n>> @@ -0,0 +1,41 @@\n>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * per_frame_controls.h - Tests for per frame controls\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +\n>> +#include \n>> +\n>> +#include \"../common/event_loop.h\"\n>> +\n>> +#include \"simple_capture.h\"\n>> +#include \"time_sheet.h\"\n>> +\n>> +class PerFrameControls : public SimpleCapture\n>> +{\n>> +public:\n>> + PerFrameControls(std::shared_ptr camera);\n>> +\n>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n>> + void runCaptureSession();\n>> +\n>> + void testFramePreciseExposureChange();\n>> + void testFramePreciseGainChange();\n>> + void testExposureGainIsAppliedOnFirstFrame();\n>> + void testExposureGainFromFirstRequestGetsApplied();\n>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n>> +\n>> + int queueRequest(libcamera::Request *request);\n>> + void requestComplete(libcamera::Request *request) override;\n>> +\n>> + unsigned int queueCount_;\n>> + unsigned int captureCount_;\n>> + unsigned int captureLimit_;\n>> +\n>> + std::weak_ptr timeSheet_;\n>> +};\n>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n>> index cf4d7cf3..56680a83 100644\n>> --- a/src/apps/lc-compliance/simple_capture.cpp\n>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n>> }\n>> }\n>> \n>> -void SimpleCapture::start()\n>> +void SimpleCapture::start(const ControlList *controls)\n>> {\n>> Stream *stream = config_->at(0).stream();\n>> int count = allocator_->allocate(stream);\n>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>> \n>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>> \n>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n>> }\n>> \n>> void SimpleCapture::stop()\n>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n>> index 2911d601..54b1d54b 100644\n>> --- a/src/apps/lc-compliance/simple_capture.h\n>> +++ b/src/apps/lc-compliance/simple_capture.h\n>> @@ -22,7 +22,7 @@ protected:\n>> SimpleCapture(std::shared_ptr camera);\n>> virtual ~SimpleCapture();\n>> \n>> - void start();\n>> + void start(const libcamera::ControlList *controls = nullptr);\n>> void stop();\n> \n> \n> I would suggest breaking out the change here that enables setting\n> controls at SimpleCapture::start() to it's own patch.\n> \n> \n>> \n>> virtual void requestComplete(libcamera::Request *request) = 0;\n>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n>> new file mode 100644\n>> index 00000000..9a0e6544\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> \n> I think introducting TimeSheet can be a separate patch to make it\n> clearer.\n> \n> \n>> @@ -0,0 +1,135 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.cpp\n>> + */\n>> +#include \"time_sheet.h\"\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \"libcamera/internal/formats.h\"\n>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>> +\n>> +using namespace libcamera;\n>> +\n>> +double calcPixelMeanNV12(const uint8_t *data)\n>> +{\n>> + return (double)*data;\n> \n> Oh, from the name \"calcPixelMean\" I was expecting a calculation, not\n> just a cast.\n\nIt was actually coincidence that I happened to start with pixel formats where a\nsimple cast is sufficient. In case of RGB it would be a calculation :-)\n\n> \n> \n> Maybe \"double readPixelNV12()\" or something.\n> \n> I think Laurent started looking at some 'image' type classes to tackle\n> something similar to this problem.\n> \n\nThat would be nice.\n\nCheers,\nStefan\n\n> \n>> +}\n>> +\n>> +double calcPixelMeanRAW10(const uint8_t *data)\n>> +{\n>> + return (double)*((const uint16_t *)data);\n>> +}\n>> +\n>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n>> +{\n>> + const Request::BufferMap &buffers = request->buffers();\n>> + for (const auto &[stream, buffer] : buffers) {\n>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n>> + if (in.isValid()) {\n>> + auto data = in.planes()[0].data();\n>> + auto streamConfig = stream->configuration();\n>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n>> +\n>> + std::function calcPixelMean;\n>> + int pixelStride;\n>> +\n>> + switch (streamConfig.pixelFormat) {\n>> + case formats::NV12:\n>> + calcPixelMean = calcPixelMeanNV12;\n>> + pixelStride = 1;\n>> + break;\n>> + case formats::SRGGB10:\n>> + calcPixelMean = calcPixelMeanRAW10;\n>> + pixelStride = 2;\n>> + break;\n>> + default:\n>> + std::stringstream s;\n>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n>> + throw std::invalid_argument(s.str());\n>> + }\n>> +\n>> + double sum = 0;\n>> + int w = 20;\n>> + int xs = streamConfig.size.width / 2 - w / 2;\n>> + int ys = streamConfig.size.height / 2 - w / 2;\n>> +\n>> + for (auto y = ys; y < ys + w; y++) {\n>> + auto line = data + y * streamConfig.stride;\n>> + for (auto x = xs; x < xs + w; x++) {\n>> + sum += calcPixelMean(line + x * pixelStride);\n>> + }\n>> + }\n>> + sum = sum / (w * w);\n>> + return sum;\n>> + }\n>> + }\n>> + return 0;\n>> +}\n>> +\n>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n>> + : controls_(idmap)\n>> +{\n>> +}\n>> +\n>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n>> +{\n>> + metadata_ = request->metadata();\n>> +\n>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n>> + if (previous) {\n>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n>> + }\n>> + sequence_ = request->sequence();\n>> +}\n>> +\n>> +void TimeSheetEntry::printInfo()\n>> +{\n>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n>> +\n>> + if (!metadata_.empty()) {\n>> + std::cout << \"Metadata:\" << std::endl;\n>> + auto idMap = metadata_.idMap();\n>> + assert(idMap);\n>> + for (const auto &[id, value] : metadata_) {\n>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n>> + }\n>> + }\n>> +}\n>> +\n>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n>> +{\n>> + auto &entry = entries_[pos];\n>> + if (!entry)\n>> + entry = std::make_shared(idmap_);\n>> + return *entry;\n>> +}\n>> +\n>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n>> +{\n>> + request->controls() = get(sequence).controls();\n>> +}\n>> +\n>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n>> +{\n>> + uint32_t sequence = request->sequence();\n>> + auto &entry = get(sequence);\n>> + TimeSheetEntry *previous = nullptr;\n>> + if (sequence >= 1) {\n>> + previous = entries_[sequence - 1].get();\n>> + }\n>> +\n>> + entry.handleCompleteRequest(request, previous);\n>> +}\n>> +\n>> +void TimeSheet::printAllInfos()\n>> +{\n>> + for (auto entry : entries_) {\n>> + if (entry)\n>> + entry->printInfo();\n>> + }\n>> +}\n>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n>> new file mode 100644\n>> index 00000000..c155763c\n>> --- /dev/null\n>> +++ b/src/apps/lc-compliance/time_sheet.h\n>> @@ -0,0 +1,53 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2024, Ideas on Board Oy\n>> + *\n>> + * time_sheet.h\n>> + */\n>> +\n>> +#pragma once\n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +\n>> +class TimeSheetEntry\n>> +{\n>> +public:\n>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n>> +\n>> + libcamera::ControlList &controls() { return controls_; };\n>> + libcamera::ControlList &metadata() { return metadata_; };\n>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n>> + void printInfo();\n>> + double getSpotBrightness() const { return spotBrightness_; };\n>> + double getBrightnessChange() const { return brightnessChange_; };\n>> +\n>> +private:\n>> + double spotBrightness_ = 0.0;\n>> + double brightnessChange_ = 0.0;\n>> + libcamera::ControlList controls_;\n>> + libcamera::ControlList metadata_;\n>> + uint32_t sequence_ = 0;\n>> +};\n>> +\n>> +class TimeSheet\n>> +{\n>> +public:\n>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n>> + : idmap_(idmap), entries_(count){};\n>> +\n>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n>> + void handleCompleteRequest(libcamera::Request *request);\n>> + void printAllInfos();\n>> +\n>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n>> + TimeSheetEntry &get(size_t pos);\n>> +\n>> +private:\n>> + const libcamera::ControlIdMap &idmap_;\n>> + std::vector> entries_;\n>> +};\n>> -- \n>> 2.40.1\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 79F45BE080\n\tfor ;\n\tFri, 1 Mar 2024 13:34:57 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A22386286F;\n\tFri, 1 Mar 2024 14:34:56 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 71CD56285F\n\tfor ;\n\tFri, 1 Mar 2024 14:34:55 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d] (unknown\n\t[IPv6:2a00:6020:448c:6c00:6a87:ef22:cc2a:689d])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 20EB79CE;\n\tFri, 1 Mar 2024 14:34:41 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"gsxIDZQt\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709300081;\n\tbh=xg1/rbZO7EcTGHNU7CNelyksld9/LkX59QcNnDYn8UU=;\n\th=Date:Subject:To:References:Cc:From:In-Reply-To:From;\n\tb=gsxIDZQt+flquKkzcyX85DRdKMHOsVHFmls55NrxxWR1jwzdSsfhYlfHQG6prhoU5\n\tYVCjrgedYARs2FxRqxyew45h+7ACDTmaMBb3TQYQLk/Z8nWsk5SesTEEDvfY+7tA6z\n\t40pZni+4QYItZJ9/RcD2QbxMD2pt6YPOZ/KygTD4=","Message-ID":"","Date":"Fri, 1 Mar 2024 14:34:52 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"Kieran Bingham ","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t<170922817185.1011926.7574189277331671166@ping.linuxembedded.co.uk>","From":"Stefan Klug ","In-Reply-To":"<170922817185.1011926.7574189277331671166@ping.linuxembedded.co.uk>","Content-Type":"text/plain; charset=UTF-8","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28819,"web_url":"https://patchwork.libcamera.org/comment/28819/","msgid":"","date":"2024-03-04T11:35:41","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Stefan\n\nOn Fri, 1 Mar 2024 at 13:22, Stefan Klug wrote:\n>\n> Hi David,\n>\n> Am 01.03.24 um 13:04 schrieb David Plowman:\n> > Hi Stefan\n> >\n> > On Fri, 1 Mar 2024 at 11:22, Stefan Klug wrote:\n> >>\n> >> Hi David,\n> >>\n> >> thanks for your comment. This is the same message I sent before, but with proper\n> >> linewrapping - sorry about that.\n> >>\n> >> Am 01.03.24 um 08:56 schrieb David Plowman:\n> >>> Hi Stefan\n> >>>\n> >>> Thanks for posting this and re-starting the discussion of per-frame controls.\n> >>>\n> >>> Could you perhaps just summarise exactly what you mean by per-frame\n> >>> controls, that is to say, what is being tested here?\n> >>>\n> >>> Eye-balling the code, I think I understood that:\n> >>>\n> >>> * You are setting controls in a specific request.\n> >>> * When that request completes, you are expecting the controls to have\n> >>> taken effect in the images that were returned with that request (but\n> >>> not earlier).\n> >>>\n> >>> But I wasn't totally sure - have I understood that correctly?\n> >>>\n> >>> Also, do we know if any other pipeline handlers implement this\n> >>> behaviour? Do folks think that all pipeline handlers should implement\n> >>> this behaviour?\n> >>>\n> >>> Thanks again!\n> >>> David\n> >>>\n> >> You are completely right. This needs a bit more context.\n> >>\n> >> It all started on my side with implementing metadata support in SimplePipeline\n> >> handler. I soon hit some corner cases where I expected things to behave\n> >> differently in cooperation with DelayedControls. So either I hit a bug in\n> >> DelayedControls or I didn't fully understand the concept behind it.\n> >>\n> >> I ended up changing several things in DelayedControls which I believe where\n> >> correct. The question then was how to prove correctness as all current users of\n> >> DelayedControls where ISP implementations. There are some unittests, but the\n> >> order of calls in these tests also felt counterintuitive (might well be, that\n> >> it's just missing knowledge on my side).\n> >>\n> >> In that area there were also no tests in lc-compliance which tested the\n> >> behaviour of an actual sensor/isp combination. So I started to write these tests\n> >> with my personal expectation of how I believe things should work. These tests\n> >> pass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\n> >> these tests and hit some corners where work would be required.\n> >>\n> >> Before digging deeper into that I believe it makes sense to see if my\n> >> expectations are correct and if a broader audience agrees to them. So here we\n> >> are, that's the reason for the RFC.\n> >>\n> >> Now on to the technical details:\n> >> Yes my expectation on per-frame-controls would be: If I queue something for\n> >> frame x, I expect that the system tries to fullfill exacty that request (and\n> >> never earlier). I'm well aware that this will not be possible in every case when\n> >> an ISP is involved, so the ISP should do the best it can, but the metadata\n> >> should reflect what was actually achieved. All the tests are currently for\n> >> manual mode, so no regulation is involved (at least for the params I test).\n> >>\n> >> Do you agree with these assumptions? Looking forward to your opinion.\n> >\n> > Thanks for the clarification, and glad that I've understood!\n> >\n> > A bit of context from our side... We looked at and implemented a\n> > per-frame controls solution for the Raspberry Pi back in summer 2022.\n> > I would say there were a couple of differences:\n> >\n> > 1. We included a mechanism for reporting failure cases. The only real\n> > failure case we have is failing to update camera exposure/gain\n> > settings in time because there is quite a short window in which to\n> > service the camera frame interrupts (especially when the system is\n> > busy and the framerate is fast).\n>\n> That's interesting. Could you explain the mechanics you used for that\n> feedback or point me to a patchset? Would be interesting to see which\n> failure cases you modelled. How did you test these?\n\nThe last time we did any work on this was last summer for a libcamera\nF2F in Prague. I think this\nhttps://github.com/naushir/libcamera/tree/pfc_update was the code we\nwere running at that time. Obviously things have moved on since then,\nnot least the existence of Pi 5, which will need handling as well.\n\nThe way the feedback worked was that you got a sequence number every\ntime you submitted a request with controls in it (distinct from the\nrequest sequence number), which we called the \"submit id\". Every\nrequest that completed contained one of these sequence numbers\nindicating which the most recent controls were that had been applied,\nand this was known as the \"sync id\". So to ask \"does this request\nimplement the controls that I set in it when I queued it?\" you would\ncheck \"requset->syncId == request->submitId\".\n\n> >\n> > The mechanism allowed the application to check whether controls had\n> > been applied as expected, or whether they'd been delayed to the next\n> > frame. It told you whether they'd got \"merged\" with the next set of\n> > controls, or whether all the controls are now running \"a frame late\".\n> >\n> > 2. We implemented our scheme in the same way as you've described,\n> > where the request where you set the controls has the first images that\n> > fulfill the request. We called this \"Android mode\" though tbh I'm not\n> > sure whether or not this really is what Android does!!!\n> >\n> > Although in the end we decided we didn't like this behaviour so much\n> > because controls get delayed to the back of all the requests that have\n> > been queued (and we normally queue quite a lot so as to avoid the risk\n>\n> Why was it necessary to move them to the back of all request? I guess in\n> a typical usecase most requests would not contain controls (at least not\n> from user side). So there would be a way to apply them as early in the\n> queue as possible. I guess I'm missing somethimng here.\n\nSorry, I'm not being super clear! We never \"moved\" controls to the\nback of the requests, the problem is that when you queue a request\nwith some controls in it, those controls are necessarily behind all\nthe other requests that you queued previously. So you have potentially\nmany extra frames of latency.\n\nBut you're exactly right, most requests don't contain any controls, so\nwe used to move the controls up the request queue so that they would\nhappen earlier.\n\n>\n> > of frame drops). So we also added \"Raspberry Pi mode\" where controls\n> > are applied earlier.\n> >\n> > In principle I would be happy to let the Pi run in either mode\n> > (\"Android\" or \"Pi\"), so long as we can choose \"Pi mode\" for our\n> > applications. In both cases the same reporting mechanism told the\n> > application exactly when the controls had been applied, so that\n> > applications could be \"mode\" agnostic.\n> >\n> > Since Summer 2022, per-frame controls have not really progressed so\n> > far as I know. We are intending to resurrect our per-frame controls\n> > implementation again once we've got over the Pi 5 libcamera\n> > integration, so I'm very happy to be resuming this discussion.\n>\n> Sure. Would be great if we could come up with a set of tests that wee\n> all agree on.\n\nI agree! Though the last 18 months suggests to me that we're not there yet.\n\nThere's currently work going on that splits buffers out of requests.\nTo me, the problem with the control lists suggests that possibly\ncontrols should have their own queues and be removed from requests as\nwell. At which point this all becomes an even bigger discussion.\n\nSo I'm kind of sorry to be listing all the ways in which per-frame\ncontrols are an unresolved problem, but not really knowing where it\ngoes from here. Nonetheless, we are wanting to move forward once the\nPi 5 integration is out of the way.\n\nDavid\n\n>\n> Cheers,\n> Stefan\n>\n> >\n> > Thanks!\n> > David\n> >\n> >>\n> >> Best regards,\n> >> Stefan\n> >>\n> >>>\n> >>> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n> >>>>\n> >>>> These tests check if controls (only exposure time and analogue gain at\n> >>>> the moment) get applied on the frame they were requested for.\n> >>>>\n> >>>> This is tested by looking at the metadata and additionally by\n> >>>> calculating a mean brightness on a centered rect of 20x20 pixels.\n> >>>>\n> >>>> Until today, these tests where only run on a project specific branch\n> >>>> with a modified simple pipeline. In theory they should pass on a\n> >>>> current master :-)\n> >>>>\n> >>>> Current test setup: imx219 with simple pipeline on an imx8mp.\n> >>>> Modifications of either the exposure delay or the gain delay in\n> >>>> the camera_sensor class resulted in test failures.\n> >>>> Which is exactly what this test shall proove.\n> >>>>\n> >>>> Signed-off-by: Stefan Klug \n> >>>> ---\n> >>>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> >>>> src/apps/lc-compliance/meson.build | 2 +\n> >>>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> >>>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> >>>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> >>>> src/apps/lc-compliance/simple_capture.h | 2 +-\n> >>>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> >>>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> >>>> 8 files changed, 487 insertions(+), 3 deletions(-)\n> >>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> >>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> >>>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> >>>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n> >>>>\n> >>>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> >>>> index 1dcfcf92..43fe59f3 100644\n> >>>> --- a/src/apps/lc-compliance/capture_test.cpp\n> >>>> +++ b/src/apps/lc-compliance/capture_test.cpp\n> >>>> @@ -11,6 +11,7 @@\n> >>>> #include \n> >>>>\n> >>>> #include \"environment.h\"\n> >>>> +#include \"per_frame_controls.h\"\n> >>>> #include \"simple_capture.h\"\n> >>>>\n> >>>> using namespace libcamera;\n> >>>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> >>>> testing::Combine(testing::ValuesIn(ROLES),\n> >>>> testing::ValuesIn(NUMREQUESTS)),\n> >>>> SingleStream::nameParameters);\n> >>>> +\n> >>>> +/*\n> >>>> + * Test Per frame controls\n> >>>> + */\n> >>>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> >>>> +{\n> >>>> + PerFrameControls capture(camera_);\n> >>>> + capture.configure(StreamRole::Viewfinder);\n> >>>> + capture.testFramePreciseExposureChange();\n> >>>> +}\n> >>>> +\n> >>>> +TEST_F(SingleStream, testFramePreciseGainChange)\n> >>>> +{\n> >>>> + PerFrameControls capture(camera_);\n> >>>> + capture.configure(StreamRole::Viewfinder);\n> >>>> + capture.testFramePreciseGainChange();\n> >>>> +}\n> >>>> +\n> >>>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> >>>> +{\n> >>>> + PerFrameControls capture(camera_);\n> >>>> + capture.configure(StreamRole::Viewfinder);\n> >>>> + capture.testExposureGainIsAppliedOnFirstFrame();\n> >>>> +}\n> >>>> +\n> >>>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> >>>> +{\n> >>>> + PerFrameControls capture(camera_);\n> >>>> + capture.configure(StreamRole::Viewfinder);\n> >>>> + capture.testExposureGainFromFirstRequestGetsApplied();\n> >>>> +}\n> >>>> +\n> >>>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> >>>> +{\n> >>>> + PerFrameControls capture(camera_);\n> >>>> + capture.configure(StreamRole::Viewfinder);\n> >>>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> >>>> +}\n> >>>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> >>>> index c792f072..2a6f52af 100644\n> >>>> --- a/src/apps/lc-compliance/meson.build\n> >>>> +++ b/src/apps/lc-compliance/meson.build\n> >>>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> >>>> 'capture_test.cpp',\n> >>>> 'environment.cpp',\n> >>>> 'main.cpp',\n> >>>> + 'per_frame_controls.cpp',\n> >>>> 'simple_capture.cpp',\n> >>>> + 'time_sheet.cpp',\n> >>>> ])\n> >>>>\n> >>>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> >>>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> >>>> new file mode 100644\n> >>>> index 00000000..70fc44ac\n> >>>> --- /dev/null\n> >>>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> >>>> @@ -0,0 +1,214 @@\n> >>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> >>>> +/*\n> >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> >>>> + *\n> >>>> + * per_frame_controls.cpp - Tests for per frame controls\n> >>>> + */\n> >>>> +#include \"per_frame_controls.h\"\n> >>>> +\n> >>>> +#include \n> >>>> +\n> >>>> +#include \"time_sheet.h\"\n> >>>> +\n> >>>> +using namespace libcamera;\n> >>>> +\n> >>>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> >>>> + : SimpleCapture(camera)\n> >>>> +{\n> >>>> +}\n> >>>> +\n> >>>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> >>>> +{\n> >>>> + ControlList ctrls(camera_->controls().idmap());\n> >>>> + /* Ensure defined default values */\n> >>>> + ctrls.set(controls::AeEnable, false);\n> >>>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> >>>> + ctrls.set(controls::ExposureTime, 10000);\n> >>>> + ctrls.set(controls::AnalogueGain, 1.0);\n> >>>> +\n> >>>> + if (controls) {\n> >>>> + ctrls.merge(*controls, true);\n> >>>> + }\n> >>>> +\n> >>>> + start(&ctrls);\n> >>>> +\n> >>>> + queueCount_ = 0;\n> >>>> + captureCount_ = 0;\n> >>>> + captureLimit_ = framesToCapture;\n> >>>> +\n> >>>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> >>>> + timeSheet_ = timeSheet;\n> >>>> + return timeSheet;\n> >>>> +}\n> >>>> +\n> >>>> +int PerFrameControls::queueRequest(Request *request)\n> >>>> +{\n> >>>> + queueCount_++;\n> >>>> + if (queueCount_ > captureLimit_)\n> >>>> + return 0;\n> >>>> +\n> >>>> + auto ts = timeSheet_.lock();\n> >>>> + if (ts) {\n> >>>> + ts->prepareForQueue(request, queueCount_ - 1);\n> >>>> + }\n> >>>> +\n> >>>> + return camera_->queueRequest(request);\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::requestComplete(Request *request)\n> >>>> +{\n> >>>> + auto ts = timeSheet_.lock();\n> >>>> + if (ts) {\n> >>>> + ts->handleCompleteRequest(request);\n> >>>> + }\n> >>>> +\n> >>>> + captureCount_++;\n> >>>> + if (captureCount_ >= captureLimit_) {\n> >>>> + loop_->exit(0);\n> >>>> + return;\n> >>>> + }\n> >>>> +\n> >>>> + request->reuse(Request::ReuseBuffers);\n> >>>> + if (queueRequest(request))\n> >>>> + loop_->exit(-EINVAL);\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::runCaptureSession()\n> >>>> +{\n> >>>> + Stream *stream = config_->at(0).stream();\n> >>>> + const std::vector> &buffers = allocator_->buffers(stream);\n> >>>> +\n> >>>> + /* Queue the recommended number of reqeuests. */\n> >>>> + for (const std::unique_ptr &buffer : buffers) {\n> >>>> + std::unique_ptr request = camera_->createRequest();\n> >>>> + request->addBuffer(stream, buffer.get());\n> >>>> + queueRequest(request.get());\n> >>>> + requests_.push_back(std::move(request));\n> >>>> + }\n> >>>> +\n> >>>> + /* Run capture session. */\n> >>>> + loop_ = new EventLoop();\n> >>>> + loop_->exec();\n> >>>> + stop();\n> >>>> + delete loop_;\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::testFramePreciseExposureChange()\n> >>>> +{\n> >>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n> >>>> + auto &ts = *timeSheet;\n> >>>> +\n> >>>> +\n> >>>> + ts[3].controls().set(controls::ExposureTime, 5000);\n> >>>> + //wait a few frames to settle\n> >>>> + ts[6].controls().set(controls::ExposureTime, 20000);\n> >>>> + ts.printAllInfos();\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> >>>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> >>>> +\n> >>>> + /* No increase just before setting exposure */\n> >>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> >>>> + /*\n> >>>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> >>>> + * This should be investigated.\n> >>>> + */\n> >>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> >>>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> >>>> +\n> >>>> + /* No increase just after setting exposure */\n> >>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> >>>> +\n> >>>> + /* No increase just after setting exposure */\n> >>>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::testFramePreciseGainChange()\n> >>>> +{\n> >>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n> >>>> + auto &ts = *timeSheet;\n> >>>> +\n> >>>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n> >>>> + //wait a few frames to settle\n> >>>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> >>>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >>>> +\n> >>>> + /* No increase just before setting gain */\n> >>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> >>>> + /*\n> >>>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> >>>> + */\n> >>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> >>>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> >>>> +\n> >>>> + /* No increase just after setting gain */\n> >>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> >>>> +{\n> >>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n> >>>> + auto &ts = *timeSheet;\n> >>>> +\n> >>>> + ts[0].controls().set(controls::ExposureTime, 10000);\n> >>>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + /* We expect it to be applied after 3 frames, the latest*/\n> >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> >>>> +{\n> >>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n> >>>> + auto &ts = *timeSheet;\n> >>>> +\n> >>>> + ts[0].controls().set(controls::ExposureTime, 8000);\n> >>>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n> >>>> + ts[1].controls().set(controls::ExposureTime, 10000);\n> >>>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + /* We expect it to be applied after 3 frames, the latest*/\n> >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> >>>> +}\n> >>>> +\n> >>>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> >>>> +{\n> >>>> + ControlList startValues;\n> >>>> + startValues.set(controls::ExposureTime, 5000);\n> >>>> + startValues.set(controls::AnalogueGain, 1.0);\n> >>>> +\n> >>>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> >>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> >>>> +\n> >>>> + /* Second capture with different values to ensure we don't hit default/old values */\n> >>>> +\n> >>>> + startValues.set(controls::ExposureTime, 15000);\n> >>>> + startValues.set(controls::AnalogueGain, 4.0);\n> >>>> +\n> >>>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> >>>> +\n> >>>> + runCaptureSession();\n> >>>> +\n> >>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> >>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> >>>> +\n> >>>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> >>>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> >>>> + EXPECT_GT(brightnessChange, 2.0);\n> >>>> +}\n> >>>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> >>>> new file mode 100644\n> >>>> index 00000000..e783f024\n> >>>> --- /dev/null\n> >>>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> >>>> @@ -0,0 +1,41 @@\n> >>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> >>>> +/*\n> >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> >>>> + *\n> >>>> + * per_frame_controls.h - Tests for per frame controls\n> >>>> + */\n> >>>> +\n> >>>> +#pragma once\n> >>>> +\n> >>>> +#include \n> >>>> +\n> >>>> +#include \n> >>>> +\n> >>>> +#include \"../common/event_loop.h\"\n> >>>> +\n> >>>> +#include \"simple_capture.h\"\n> >>>> +#include \"time_sheet.h\"\n> >>>> +\n> >>>> +class PerFrameControls : public SimpleCapture\n> >>>> +{\n> >>>> +public:\n> >>>> + PerFrameControls(std::shared_ptr camera);\n> >>>> +\n> >>>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> >>>> + void runCaptureSession();\n> >>>> +\n> >>>> + void testFramePreciseExposureChange();\n> >>>> + void testFramePreciseGainChange();\n> >>>> + void testExposureGainIsAppliedOnFirstFrame();\n> >>>> + void testExposureGainFromFirstRequestGetsApplied();\n> >>>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n> >>>> +\n> >>>> + int queueRequest(libcamera::Request *request);\n> >>>> + void requestComplete(libcamera::Request *request) override;\n> >>>> +\n> >>>> + unsigned int queueCount_;\n> >>>> + unsigned int captureCount_;\n> >>>> + unsigned int captureLimit_;\n> >>>> +\n> >>>> + std::weak_ptr timeSheet_;\n> >>>> +};\n> >>>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> >>>> index cf4d7cf3..56680a83 100644\n> >>>> --- a/src/apps/lc-compliance/simple_capture.cpp\n> >>>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> >>>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> >>>> }\n> >>>> }\n> >>>>\n> >>>> -void SimpleCapture::start()\n> >>>> +void SimpleCapture::start(const ControlList *controls)\n> >>>> {\n> >>>> Stream *stream = config_->at(0).stream();\n> >>>> int count = allocator_->allocate(stream);\n> >>>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n> >>>>\n> >>>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> >>>>\n> >>>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> >>>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> >>>> }\n> >>>>\n> >>>> void SimpleCapture::stop()\n> >>>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> >>>> index 2911d601..54b1d54b 100644\n> >>>> --- a/src/apps/lc-compliance/simple_capture.h\n> >>>> +++ b/src/apps/lc-compliance/simple_capture.h\n> >>>> @@ -22,7 +22,7 @@ protected:\n> >>>> SimpleCapture(std::shared_ptr camera);\n> >>>> virtual ~SimpleCapture();\n> >>>>\n> >>>> - void start();\n> >>>> + void start(const libcamera::ControlList *controls = nullptr);\n> >>>> void stop();\n> >>>>\n> >>>> virtual void requestComplete(libcamera::Request *request) = 0;\n> >>>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> >>>> new file mode 100644\n> >>>> index 00000000..9a0e6544\n> >>>> --- /dev/null\n> >>>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> >>>> @@ -0,0 +1,135 @@\n> >>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> >>>> +/*\n> >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> >>>> + *\n> >>>> + * time_sheet.cpp\n> >>>> + */\n> >>>> +#include \"time_sheet.h\"\n> >>>> +\n> >>>> +#include \n> >>>> +#include \n> >>>> +\n> >>>> +#include \"libcamera/internal/formats.h\"\n> >>>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> >>>> +\n> >>>> +using namespace libcamera;\n> >>>> +\n> >>>> +double calcPixelMeanNV12(const uint8_t *data)\n> >>>> +{\n> >>>> + return (double)*data;\n> >>>> +}\n> >>>> +\n> >>>> +double calcPixelMeanRAW10(const uint8_t *data)\n> >>>> +{\n> >>>> + return (double)*((const uint16_t *)data);\n> >>>> +}\n> >>>> +\n> >>>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> >>>> +{\n> >>>> + const Request::BufferMap &buffers = request->buffers();\n> >>>> + for (const auto &[stream, buffer] : buffers) {\n> >>>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> >>>> + if (in.isValid()) {\n> >>>> + auto data = in.planes()[0].data();\n> >>>> + auto streamConfig = stream->configuration();\n> >>>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> >>>> +\n> >>>> + std::function calcPixelMean;\n> >>>> + int pixelStride;\n> >>>> +\n> >>>> + switch (streamConfig.pixelFormat) {\n> >>>> + case formats::NV12:\n> >>>> + calcPixelMean = calcPixelMeanNV12;\n> >>>> + pixelStride = 1;\n> >>>> + break;\n> >>>> + case formats::SRGGB10:\n> >>>> + calcPixelMean = calcPixelMeanRAW10;\n> >>>> + pixelStride = 2;\n> >>>> + break;\n> >>>> + default:\n> >>>> + std::stringstream s;\n> >>>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n> >>>> + throw std::invalid_argument(s.str());\n> >>>> + }\n> >>>> +\n> >>>> + double sum = 0;\n> >>>> + int w = 20;\n> >>>> + int xs = streamConfig.size.width / 2 - w / 2;\n> >>>> + int ys = streamConfig.size.height / 2 - w / 2;\n> >>>> +\n> >>>> + for (auto y = ys; y < ys + w; y++) {\n> >>>> + auto line = data + y * streamConfig.stride;\n> >>>> + for (auto x = xs; x < xs + w; x++) {\n> >>>> + sum += calcPixelMean(line + x * pixelStride);\n> >>>> + }\n> >>>> + }\n> >>>> + sum = sum / (w * w);\n> >>>> + return sum;\n> >>>> + }\n> >>>> + }\n> >>>> + return 0;\n> >>>> +}\n> >>>> +\n> >>>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> >>>> + : controls_(idmap)\n> >>>> +{\n> >>>> +}\n> >>>> +\n> >>>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> >>>> +{\n> >>>> + metadata_ = request->metadata();\n> >>>> +\n> >>>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> >>>> + if (previous) {\n> >>>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> >>>> + }\n> >>>> + sequence_ = request->sequence();\n> >>>> +}\n> >>>> +\n> >>>> +void TimeSheetEntry::printInfo()\n> >>>> +{\n> >>>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n> >>>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> >>>> +\n> >>>> + if (!metadata_.empty()) {\n> >>>> + std::cout << \"Metadata:\" << std::endl;\n> >>>> + auto idMap = metadata_.idMap();\n> >>>> + assert(idMap);\n> >>>> + for (const auto &[id, value] : metadata_) {\n> >>>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> >>>> + }\n> >>>> + }\n> >>>> +}\n> >>>> +\n> >>>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> >>>> +{\n> >>>> + auto &entry = entries_[pos];\n> >>>> + if (!entry)\n> >>>> + entry = std::make_shared(idmap_);\n> >>>> + return *entry;\n> >>>> +}\n> >>>> +\n> >>>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> >>>> +{\n> >>>> + request->controls() = get(sequence).controls();\n> >>>> +}\n> >>>> +\n> >>>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> >>>> +{\n> >>>> + uint32_t sequence = request->sequence();\n> >>>> + auto &entry = get(sequence);\n> >>>> + TimeSheetEntry *previous = nullptr;\n> >>>> + if (sequence >= 1) {\n> >>>> + previous = entries_[sequence - 1].get();\n> >>>> + }\n> >>>> +\n> >>>> + entry.handleCompleteRequest(request, previous);\n> >>>> +}\n> >>>> +\n> >>>> +void TimeSheet::printAllInfos()\n> >>>> +{\n> >>>> + for (auto entry : entries_) {\n> >>>> + if (entry)\n> >>>> + entry->printInfo();\n> >>>> + }\n> >>>> +}\n> >>>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> >>>> new file mode 100644\n> >>>> index 00000000..c155763c\n> >>>> --- /dev/null\n> >>>> +++ b/src/apps/lc-compliance/time_sheet.h\n> >>>> @@ -0,0 +1,53 @@\n> >>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> >>>> +/*\n> >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> >>>> + *\n> >>>> + * time_sheet.h\n> >>>> + */\n> >>>> +\n> >>>> +#pragma once\n> >>>> +\n> >>>> +#include \n> >>>> +#include \n> >>>> +\n> >>>> +#include \n> >>>> +\n> >>>> +class TimeSheetEntry\n> >>>> +{\n> >>>> +public:\n> >>>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> >>>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> >>>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n> >>>> +\n> >>>> + libcamera::ControlList &controls() { return controls_; };\n> >>>> + libcamera::ControlList &metadata() { return metadata_; };\n> >>>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> >>>> + void printInfo();\n> >>>> + double getSpotBrightness() const { return spotBrightness_; };\n> >>>> + double getBrightnessChange() const { return brightnessChange_; };\n> >>>> +\n> >>>> +private:\n> >>>> + double spotBrightness_ = 0.0;\n> >>>> + double brightnessChange_ = 0.0;\n> >>>> + libcamera::ControlList controls_;\n> >>>> + libcamera::ControlList metadata_;\n> >>>> + uint32_t sequence_ = 0;\n> >>>> +};\n> >>>> +\n> >>>> +class TimeSheet\n> >>>> +{\n> >>>> +public:\n> >>>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> >>>> + : idmap_(idmap), entries_(count){};\n> >>>> +\n> >>>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> >>>> + void handleCompleteRequest(libcamera::Request *request);\n> >>>> + void printAllInfos();\n> >>>> +\n> >>>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> >>>> + TimeSheetEntry &get(size_t pos);\n> >>>> +\n> >>>> +private:\n> >>>> + const libcamera::ControlIdMap &idmap_;\n> >>>> + std::vector> entries_;\n> >>>> +};\n> >>>> --\n> >>>> 2.40.1\n> >>>>\n> >>\n> >> --\n> >> Regards,\n> >>\n> >> Stefan Klug\n> >>\n>\n> --\n> Regards,\n>\n> Stefan Klug\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 06247BD160\n\tfor ;\n\tMon, 4 Mar 2024 11:35:57 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 295FA6286F;\n\tMon, 4 Mar 2024 12:35:56 +0100 (CET)","from mail-ua1-x92a.google.com (mail-ua1-x92a.google.com\n\t[IPv6:2607:f8b0:4864:20::92a])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 836FA627FC\n\tfor ;\n\tMon, 4 Mar 2024 12:35:53 +0100 (CET)","by mail-ua1-x92a.google.com with SMTP id\n\ta1e0cc1a2514c-7d5bbbe5844so2318293241.0\n\tfor ;\n\tMon, 04 Mar 2024 03:35:53 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"N4mtwesG\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1709552152; x=1710156952;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=sKXmnsuNUEXm53WnPbeZvERKqEEsA9QpvDHXN1Mg6/8=;\n\tb=N4mtwesG7yC0JM1XK5mjttPtb1oZ98vY22RH8X23TCsgE0PK7u1yIsJQeMqXb7qlEq\n\t1jhYHujBMrJuzFOxcJoFtBSos60oXBKLZ5FL70/J/jyYdl3PzkCy0GCzaRGIgb+nQyY2\n\t7i4f3BhGxZnMH9ZNu0Qm8IB7Lj339Ug5tKou5+SK7ZOXE7jaKJ31KJBqpkaT0kGwVajg\n\tTLVyWRYNH0jcfY4/sMDAJ7kIPxUEpelXoE1gQy8Rw/Au3l4R46xhmOdGgganbJX6mcG3\n\tDcBInLqU1XK2AQEZw8Z+lhPm6yIEzQf6ZL3CoSdtZnwtA1AufS05jDWDEsk0j83BTOpq\n\tanrQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1709552152; x=1710156952;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=sKXmnsuNUEXm53WnPbeZvERKqEEsA9QpvDHXN1Mg6/8=;\n\tb=XGebvNrouMl+RfupGPH6cwj7EtCcjHlKaybamVKhYxtwZc3s06Sr6w5sPIkGmeWvjq\n\tNsrrd/YNiR8rEv9SwoVSsAQyp4lPYsWQ/W17yiT2Y9A67vv14OIOh42XiQmzzGgaVVZJ\n\tRq+KJfx3mvIAAaNe3a5lprfp5K5CyVR+kyGmJ0CTnMrA1bkFQ9Ir2FfAyUtp/ZSrx14H\n\tqZmtvbjGqAq9i+ZITPzPN++EC7I9xyPontz6KZxMPq9tuPkKXZlxAhiOHpZXCXC5NqqS\n\tyAGzG2uAwCdK38lZu+w6VI+3GchnuWYPNCB0fAsDwDR2vpDhZPwPn5uHRdBF+9l1S4Cs\n\twdgQ==","X-Gm-Message-State":"AOJu0Yy1iucsLtA/nbcy2twuwBpVr+LQNdBbB/gwRmATJJAzc0iFBu44\n\t0pU+B62EGqc4rfdH3I+EghGK8d9A1Al5F2YfTVSJwP5HY78P16NFXiwHB/tx/YTekf7cVU/irTT\n\t0bJpVTKQ798172UPmPYEoCY2zpehF3kaxhqx7nSjpOdDBGK1gBHc=","X-Google-Smtp-Source":"AGHT+IEKzuxXFPof8mjLPOWHG3dNd4kG4w+yhkE/kwnY8npDMhQxGiLvW42coOf74q2AIGFr9IsfTNgr2S6y/KCaTmY=","X-Received":"by 2002:a67:fdc2:0:b0:472:78c4:8d73 with SMTP id\n\tl2-20020a67fdc2000000b0047278c48d73mr5340327vsq.9.1709552152128;\n\tMon, 04 Mar 2024 03:35:52 -0800 (PST)","MIME-Version":"1.0","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t\n\t\n\t\n\t<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>","In-Reply-To":"<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>","From":"David Plowman ","Date":"Mon, 4 Mar 2024 11:35:41 +0000","Message-ID":"","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","To":"Stefan Klug ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28820,"web_url":"https://patchwork.libcamera.org/comment/28820/","msgid":"<170955351406.1676185.13619836915117809277@ping.linuxembedded.co.uk>","date":"2024-03-04T11:58:34","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting David Plowman (2024-03-04 11:35:41)\n> Hi Stefan\n> \n> On Fri, 1 Mar 2024 at 13:22, Stefan Klug wrote:\n> >\n> > Hi David,\n> >\n> > Am 01.03.24 um 13:04 schrieb David Plowman:\n> > > Hi Stefan\n> > >\n> > > On Fri, 1 Mar 2024 at 11:22, Stefan Klug wrote:\n> > >>\n> > >> Hi David,\n> > >>\n> > >> thanks for your comment. This is the same message I sent before, but with proper\n> > >> linewrapping - sorry about that.\n> > >>\n> > >> Am 01.03.24 um 08:56 schrieb David Plowman:\n> > >>> Hi Stefan\n> > >>>\n> > >>> Thanks for posting this and re-starting the discussion of per-frame controls.\n> > >>>\n> > >>> Could you perhaps just summarise exactly what you mean by per-frame\n> > >>> controls, that is to say, what is being tested here?\n> > >>>\n> > >>> Eye-balling the code, I think I understood that:\n> > >>>\n> > >>> * You are setting controls in a specific request.\n> > >>> * When that request completes, you are expecting the controls to have\n> > >>> taken effect in the images that were returned with that request (but\n> > >>> not earlier).\n> > >>>\n> > >>> But I wasn't totally sure - have I understood that correctly?\n> > >>>\n> > >>> Also, do we know if any other pipeline handlers implement this\n> > >>> behaviour? Do folks think that all pipeline handlers should implement\n> > >>> this behaviour?\n> > >>>\n> > >>> Thanks again!\n> > >>> David\n> > >>>\n> > >> You are completely right. This needs a bit more context.\n> > >>\n> > >> It all started on my side with implementing metadata support in SimplePipeline\n> > >> handler. I soon hit some corner cases where I expected things to behave\n> > >> differently in cooperation with DelayedControls. So either I hit a bug in\n> > >> DelayedControls or I didn't fully understand the concept behind it.\n> > >>\n> > >> I ended up changing several things in DelayedControls which I believe where\n> > >> correct. The question then was how to prove correctness as all current users of\n> > >> DelayedControls where ISP implementations. There are some unittests, but the\n> > >> order of calls in these tests also felt counterintuitive (might well be, that\n> > >> it's just missing knowledge on my side).\n> > >>\n> > >> In that area there were also no tests in lc-compliance which tested the\n> > >> behaviour of an actual sensor/isp combination. So I started to write these tests\n> > >> with my personal expectation of how I believe things should work. These tests\n> > >> pass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\n> > >> these tests and hit some corners where work would be required.\n> > >>\n> > >> Before digging deeper into that I believe it makes sense to see if my\n> > >> expectations are correct and if a broader audience agrees to them. So here we\n> > >> are, that's the reason for the RFC.\n> > >>\n> > >> Now on to the technical details:\n> > >> Yes my expectation on per-frame-controls would be: If I queue something for\n> > >> frame x, I expect that the system tries to fullfill exacty that request (and\n> > >> never earlier). I'm well aware that this will not be possible in every case when\n> > >> an ISP is involved, so the ISP should do the best it can, but the metadata\n> > >> should reflect what was actually achieved. All the tests are currently for\n> > >> manual mode, so no regulation is involved (at least for the params I test).\n> > >>\n> > >> Do you agree with these assumptions? Looking forward to your opinion.\n> > >\n> > > Thanks for the clarification, and glad that I've understood!\n> > >\n> > > A bit of context from our side... We looked at and implemented a\n> > > per-frame controls solution for the Raspberry Pi back in summer 2022.\n> > > I would say there were a couple of differences:\n> > >\n> > > 1. We included a mechanism for reporting failure cases. The only real\n> > > failure case we have is failing to update camera exposure/gain\n> > > settings in time because there is quite a short window in which to\n> > > service the camera frame interrupts (especially when the system is\n> > > busy and the framerate is fast).\n> >\n> > That's interesting. Could you explain the mechanics you used for that\n> > feedback or point me to a patchset? Would be interesting to see which\n> > failure cases you modelled. How did you test these?\n> \n> The last time we did any work on this was last summer for a libcamera\n> F2F in Prague. I think this\n> https://github.com/naushir/libcamera/tree/pfc_update was the code we\n> were running at that time. Obviously things have moved on since then,\n> not least the existence of Pi 5, which will need handling as well.\n> \n> The way the feedback worked was that you got a sequence number every\n> time you submitted a request with controls in it (distinct from the\n> request sequence number), which we called the \"submit id\". Every\n> request that completed contained one of these sequence numbers\n> indicating which the most recent controls were that had been applied,\n> and this was known as the \"sync id\". So to ask \"does this request\n> implement the controls that I set in it when I queued it?\" you would\n> check \"requset->syncId == request->submitId\".\n> \n> > >\n> > > The mechanism allowed the application to check whether controls had\n> > > been applied as expected, or whether they'd been delayed to the next\n> > > frame. It told you whether they'd got \"merged\" with the next set of\n> > > controls, or whether all the controls are now running \"a frame late\".\n> > >\n> > > 2. We implemented our scheme in the same way as you've described,\n> > > where the request where you set the controls has the first images that\n> > > fulfill the request. We called this \"Android mode\" though tbh I'm not\n> > > sure whether or not this really is what Android does!!!\n> > >\n> > > Although in the end we decided we didn't like this behaviour so much\n> > > because controls get delayed to the back of all the requests that have\n> > > been queued (and we normally queue quite a lot so as to avoid the risk\n> >\n> > Why was it necessary to move them to the back of all request? I guess in\n> > a typical usecase most requests would not contain controls (at least not\n> > from user side). So there would be a way to apply them as early in the\n> > queue as possible. I guess I'm missing somethimng here.\n> \n> Sorry, I'm not being super clear! We never \"moved\" controls to the\n> back of the requests, the problem is that when you queue a request\n> with some controls in it, those controls are necessarily behind all\n> the other requests that you queued previously. So you have potentially\n> many extra frames of latency.\n> \n> But you're exactly right, most requests don't contain any controls, so\n> we used to move the controls up the request queue so that they would\n> happen earlier.\n> \n> >\n> > > of frame drops). So we also added \"Raspberry Pi mode\" where controls\n> > > are applied earlier.\n> > >\n> > > In principle I would be happy to let the Pi run in either mode\n> > > (\"Android\" or \"Pi\"), so long as we can choose \"Pi mode\" for our\n> > > applications. In both cases the same reporting mechanism told the\n> > > application exactly when the controls had been applied, so that\n> > > applications could be \"mode\" agnostic.\n> > >\n> > > Since Summer 2022, per-frame controls have not really progressed so\n> > > far as I know. We are intending to resurrect our per-frame controls\n> > > implementation again once we've got over the Pi 5 libcamera\n> > > integration, so I'm very happy to be resuming this discussion.\n> >\n> > Sure. Would be great if we could come up with a set of tests that wee\n> > all agree on.\n> \n> I agree! Though the last 18 months suggests to me that we're not there yet.\n\nIndeed, the whole topic hasn't had enough attention or development, I\nhope that will get more attention soon.\n\n> There's currently work going on that splits buffers out of requests.\n> To me, the problem with the control lists suggests that possibly\n> controls should have their own queues and be removed from requests as\n> well. At which point this all becomes an even bigger discussion.\n> \n> So I'm kind of sorry to be listing all the ways in which per-frame\n> controls are an unresolved problem, but not really knowing where it\n> goes from here. Nonetheless, we are wanting to move forward once the\n> Pi 5 integration is out of the way.\n\nLikewise!.\n\nI actually envisaged this work here being more about validating that\nDelayedControls applies controls correctly at the right frames, and that\nthe delays given for a specific sensor were correct. But it's quickly\nbecoming more involved and the 'per-frame' topic grows ;-)\n\n--\nKieran\n\n\n> \n> David\n> \n> >\n> > Cheers,\n> > Stefan\n> >\n> > >\n> > > Thanks!\n> > > David\n> > >\n> > >>\n> > >> Best regards,\n> > >> Stefan\n> > >>\n> > >>>\n> > >>> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n> > >>>>\n> > >>>> These tests check if controls (only exposure time and analogue gain at\n> > >>>> the moment) get applied on the frame they were requested for.\n> > >>>>\n> > >>>> This is tested by looking at the metadata and additionally by\n> > >>>> calculating a mean brightness on a centered rect of 20x20 pixels.\n> > >>>>\n> > >>>> Until today, these tests where only run on a project specific branch\n> > >>>> with a modified simple pipeline. In theory they should pass on a\n> > >>>> current master :-)\n> > >>>>\n> > >>>> Current test setup: imx219 with simple pipeline on an imx8mp.\n> > >>>> Modifications of either the exposure delay or the gain delay in\n> > >>>> the camera_sensor class resulted in test failures.\n> > >>>> Which is exactly what this test shall proove.\n> > >>>>\n> > >>>> Signed-off-by: Stefan Klug \n> > >>>> ---\n> > >>>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n> > >>>> src/apps/lc-compliance/meson.build | 2 +\n> > >>>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n> > >>>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n> > >>>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n> > >>>> src/apps/lc-compliance/simple_capture.h | 2 +-\n> > >>>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n> > >>>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n> > >>>> 8 files changed, 487 insertions(+), 3 deletions(-)\n> > >>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n> > >>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n> > >>>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n> > >>>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n> > >>>>\n> > >>>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n> > >>>> index 1dcfcf92..43fe59f3 100644\n> > >>>> --- a/src/apps/lc-compliance/capture_test.cpp\n> > >>>> +++ b/src/apps/lc-compliance/capture_test.cpp\n> > >>>> @@ -11,6 +11,7 @@\n> > >>>> #include \n> > >>>>\n> > >>>> #include \"environment.h\"\n> > >>>> +#include \"per_frame_controls.h\"\n> > >>>> #include \"simple_capture.h\"\n> > >>>>\n> > >>>> using namespace libcamera;\n> > >>>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n> > >>>> testing::Combine(testing::ValuesIn(ROLES),\n> > >>>> testing::ValuesIn(NUMREQUESTS)),\n> > >>>> SingleStream::nameParameters);\n> > >>>> +\n> > >>>> +/*\n> > >>>> + * Test Per frame controls\n> > >>>> + */\n> > >>>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n> > >>>> +{\n> > >>>> + PerFrameControls capture(camera_);\n> > >>>> + capture.configure(StreamRole::Viewfinder);\n> > >>>> + capture.testFramePreciseExposureChange();\n> > >>>> +}\n> > >>>> +\n> > >>>> +TEST_F(SingleStream, testFramePreciseGainChange)\n> > >>>> +{\n> > >>>> + PerFrameControls capture(camera_);\n> > >>>> + capture.configure(StreamRole::Viewfinder);\n> > >>>> + capture.testFramePreciseGainChange();\n> > >>>> +}\n> > >>>> +\n> > >>>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n> > >>>> +{\n> > >>>> + PerFrameControls capture(camera_);\n> > >>>> + capture.configure(StreamRole::Viewfinder);\n> > >>>> + capture.testExposureGainIsAppliedOnFirstFrame();\n> > >>>> +}\n> > >>>> +\n> > >>>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n> > >>>> +{\n> > >>>> + PerFrameControls capture(camera_);\n> > >>>> + capture.configure(StreamRole::Viewfinder);\n> > >>>> + capture.testExposureGainFromFirstRequestGetsApplied();\n> > >>>> +}\n> > >>>> +\n> > >>>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n> > >>>> +{\n> > >>>> + PerFrameControls capture(camera_);\n> > >>>> + capture.configure(StreamRole::Viewfinder);\n> > >>>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n> > >>>> +}\n> > >>>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n> > >>>> index c792f072..2a6f52af 100644\n> > >>>> --- a/src/apps/lc-compliance/meson.build\n> > >>>> +++ b/src/apps/lc-compliance/meson.build\n> > >>>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n> > >>>> 'capture_test.cpp',\n> > >>>> 'environment.cpp',\n> > >>>> 'main.cpp',\n> > >>>> + 'per_frame_controls.cpp',\n> > >>>> 'simple_capture.cpp',\n> > >>>> + 'time_sheet.cpp',\n> > >>>> ])\n> > >>>>\n> > >>>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n> > >>>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n> > >>>> new file mode 100644\n> > >>>> index 00000000..70fc44ac\n> > >>>> --- /dev/null\n> > >>>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n> > >>>> @@ -0,0 +1,214 @@\n> > >>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> > >>>> +/*\n> > >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> > >>>> + *\n> > >>>> + * per_frame_controls.cpp - Tests for per frame controls\n> > >>>> + */\n> > >>>> +#include \"per_frame_controls.h\"\n> > >>>> +\n> > >>>> +#include \n> > >>>> +\n> > >>>> +#include \"time_sheet.h\"\n> > >>>> +\n> > >>>> +using namespace libcamera;\n> > >>>> +\n> > >>>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n> > >>>> + : SimpleCapture(camera)\n> > >>>> +{\n> > >>>> +}\n> > >>>> +\n> > >>>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n> > >>>> +{\n> > >>>> + ControlList ctrls(camera_->controls().idmap());\n> > >>>> + /* Ensure defined default values */\n> > >>>> + ctrls.set(controls::AeEnable, false);\n> > >>>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n> > >>>> + ctrls.set(controls::ExposureTime, 10000);\n> > >>>> + ctrls.set(controls::AnalogueGain, 1.0);\n> > >>>> +\n> > >>>> + if (controls) {\n> > >>>> + ctrls.merge(*controls, true);\n> > >>>> + }\n> > >>>> +\n> > >>>> + start(&ctrls);\n> > >>>> +\n> > >>>> + queueCount_ = 0;\n> > >>>> + captureCount_ = 0;\n> > >>>> + captureLimit_ = framesToCapture;\n> > >>>> +\n> > >>>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n> > >>>> + timeSheet_ = timeSheet;\n> > >>>> + return timeSheet;\n> > >>>> +}\n> > >>>> +\n> > >>>> +int PerFrameControls::queueRequest(Request *request)\n> > >>>> +{\n> > >>>> + queueCount_++;\n> > >>>> + if (queueCount_ > captureLimit_)\n> > >>>> + return 0;\n> > >>>> +\n> > >>>> + auto ts = timeSheet_.lock();\n> > >>>> + if (ts) {\n> > >>>> + ts->prepareForQueue(request, queueCount_ - 1);\n> > >>>> + }\n> > >>>> +\n> > >>>> + return camera_->queueRequest(request);\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::requestComplete(Request *request)\n> > >>>> +{\n> > >>>> + auto ts = timeSheet_.lock();\n> > >>>> + if (ts) {\n> > >>>> + ts->handleCompleteRequest(request);\n> > >>>> + }\n> > >>>> +\n> > >>>> + captureCount_++;\n> > >>>> + if (captureCount_ >= captureLimit_) {\n> > >>>> + loop_->exit(0);\n> > >>>> + return;\n> > >>>> + }\n> > >>>> +\n> > >>>> + request->reuse(Request::ReuseBuffers);\n> > >>>> + if (queueRequest(request))\n> > >>>> + loop_->exit(-EINVAL);\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::runCaptureSession()\n> > >>>> +{\n> > >>>> + Stream *stream = config_->at(0).stream();\n> > >>>> + const std::vector> &buffers = allocator_->buffers(stream);\n> > >>>> +\n> > >>>> + /* Queue the recommended number of reqeuests. */\n> > >>>> + for (const std::unique_ptr &buffer : buffers) {\n> > >>>> + std::unique_ptr request = camera_->createRequest();\n> > >>>> + request->addBuffer(stream, buffer.get());\n> > >>>> + queueRequest(request.get());\n> > >>>> + requests_.push_back(std::move(request));\n> > >>>> + }\n> > >>>> +\n> > >>>> + /* Run capture session. */\n> > >>>> + loop_ = new EventLoop();\n> > >>>> + loop_->exec();\n> > >>>> + stop();\n> > >>>> + delete loop_;\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::testFramePreciseExposureChange()\n> > >>>> +{\n> > >>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n> > >>>> + auto &ts = *timeSheet;\n> > >>>> +\n> > >>>> +\n> > >>>> + ts[3].controls().set(controls::ExposureTime, 5000);\n> > >>>> + //wait a few frames to settle\n> > >>>> + ts[6].controls().set(controls::ExposureTime, 20000);\n> > >>>> + ts.printAllInfos();\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> > >>>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n> > >>>> +\n> > >>>> + /* No increase just before setting exposure */\n> > >>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> > >>>> + /*\n> > >>>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n> > >>>> + * This should be investigated.\n> > >>>> + */\n> > >>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> > >>>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> > >>>> +\n> > >>>> + /* No increase just after setting exposure */\n> > >>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> > >>>> +\n> > >>>> + /* No increase just after setting exposure */\n> > >>>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::testFramePreciseGainChange()\n> > >>>> +{\n> > >>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n> > >>>> + auto &ts = *timeSheet;\n> > >>>> +\n> > >>>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n> > >>>> + //wait a few frames to settle\n> > >>>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n> > >>>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> > >>>> +\n> > >>>> + /* No increase just before setting gain */\n> > >>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n> > >>>> + /*\n> > >>>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n> > >>>> + */\n> > >>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n> > >>>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n> > >>>> +\n> > >>>> + /* No increase just after setting gain */\n> > >>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n> > >>>> +{\n> > >>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n> > >>>> + auto &ts = *timeSheet;\n> > >>>> +\n> > >>>> + ts[0].controls().set(controls::ExposureTime, 10000);\n> > >>>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + /* We expect it to be applied after 3 frames, the latest*/\n> > >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> > >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n> > >>>> +{\n> > >>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n> > >>>> + auto &ts = *timeSheet;\n> > >>>> +\n> > >>>> + ts[0].controls().set(controls::ExposureTime, 8000);\n> > >>>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n> > >>>> + ts[1].controls().set(controls::ExposureTime, 10000);\n> > >>>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + /* We expect it to be applied after 3 frames, the latest*/\n> > >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n> > >>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n> > >>>> +}\n> > >>>> +\n> > >>>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n> > >>>> +{\n> > >>>> + ControlList startValues;\n> > >>>> + startValues.set(controls::ExposureTime, 5000);\n> > >>>> + startValues.set(controls::AnalogueGain, 1.0);\n> > >>>> +\n> > >>>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n> > >>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n> > >>>> +\n> > >>>> + /* Second capture with different values to ensure we don't hit default/old values */\n> > >>>> +\n> > >>>> + startValues.set(controls::ExposureTime, 15000);\n> > >>>> + startValues.set(controls::AnalogueGain, 4.0);\n> > >>>> +\n> > >>>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n> > >>>> +\n> > >>>> + runCaptureSession();\n> > >>>> +\n> > >>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n> > >>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n> > >>>> +\n> > >>>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n> > >>>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n> > >>>> + EXPECT_GT(brightnessChange, 2.0);\n> > >>>> +}\n> > >>>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n> > >>>> new file mode 100644\n> > >>>> index 00000000..e783f024\n> > >>>> --- /dev/null\n> > >>>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n> > >>>> @@ -0,0 +1,41 @@\n> > >>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> > >>>> +/*\n> > >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> > >>>> + *\n> > >>>> + * per_frame_controls.h - Tests for per frame controls\n> > >>>> + */\n> > >>>> +\n> > >>>> +#pragma once\n> > >>>> +\n> > >>>> +#include \n> > >>>> +\n> > >>>> +#include \n> > >>>> +\n> > >>>> +#include \"../common/event_loop.h\"\n> > >>>> +\n> > >>>> +#include \"simple_capture.h\"\n> > >>>> +#include \"time_sheet.h\"\n> > >>>> +\n> > >>>> +class PerFrameControls : public SimpleCapture\n> > >>>> +{\n> > >>>> +public:\n> > >>>> + PerFrameControls(std::shared_ptr camera);\n> > >>>> +\n> > >>>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n> > >>>> + void runCaptureSession();\n> > >>>> +\n> > >>>> + void testFramePreciseExposureChange();\n> > >>>> + void testFramePreciseGainChange();\n> > >>>> + void testExposureGainIsAppliedOnFirstFrame();\n> > >>>> + void testExposureGainFromFirstRequestGetsApplied();\n> > >>>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n> > >>>> +\n> > >>>> + int queueRequest(libcamera::Request *request);\n> > >>>> + void requestComplete(libcamera::Request *request) override;\n> > >>>> +\n> > >>>> + unsigned int queueCount_;\n> > >>>> + unsigned int captureCount_;\n> > >>>> + unsigned int captureLimit_;\n> > >>>> +\n> > >>>> + std::weak_ptr timeSheet_;\n> > >>>> +};\n> > >>>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n> > >>>> index cf4d7cf3..56680a83 100644\n> > >>>> --- a/src/apps/lc-compliance/simple_capture.cpp\n> > >>>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n> > >>>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n> > >>>> }\n> > >>>> }\n> > >>>>\n> > >>>> -void SimpleCapture::start()\n> > >>>> +void SimpleCapture::start(const ControlList *controls)\n> > >>>> {\n> > >>>> Stream *stream = config_->at(0).stream();\n> > >>>> int count = allocator_->allocate(stream);\n> > >>>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n> > >>>>\n> > >>>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n> > >>>>\n> > >>>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n> > >>>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n> > >>>> }\n> > >>>>\n> > >>>> void SimpleCapture::stop()\n> > >>>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n> > >>>> index 2911d601..54b1d54b 100644\n> > >>>> --- a/src/apps/lc-compliance/simple_capture.h\n> > >>>> +++ b/src/apps/lc-compliance/simple_capture.h\n> > >>>> @@ -22,7 +22,7 @@ protected:\n> > >>>> SimpleCapture(std::shared_ptr camera);\n> > >>>> virtual ~SimpleCapture();\n> > >>>>\n> > >>>> - void start();\n> > >>>> + void start(const libcamera::ControlList *controls = nullptr);\n> > >>>> void stop();\n> > >>>>\n> > >>>> virtual void requestComplete(libcamera::Request *request) = 0;\n> > >>>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n> > >>>> new file mode 100644\n> > >>>> index 00000000..9a0e6544\n> > >>>> --- /dev/null\n> > >>>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n> > >>>> @@ -0,0 +1,135 @@\n> > >>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > >>>> +/*\n> > >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> > >>>> + *\n> > >>>> + * time_sheet.cpp\n> > >>>> + */\n> > >>>> +#include \"time_sheet.h\"\n> > >>>> +\n> > >>>> +#include \n> > >>>> +#include \n> > >>>> +\n> > >>>> +#include \"libcamera/internal/formats.h\"\n> > >>>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> > >>>> +\n> > >>>> +using namespace libcamera;\n> > >>>> +\n> > >>>> +double calcPixelMeanNV12(const uint8_t *data)\n> > >>>> +{\n> > >>>> + return (double)*data;\n> > >>>> +}\n> > >>>> +\n> > >>>> +double calcPixelMeanRAW10(const uint8_t *data)\n> > >>>> +{\n> > >>>> + return (double)*((const uint16_t *)data);\n> > >>>> +}\n> > >>>> +\n> > >>>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n> > >>>> +{\n> > >>>> + const Request::BufferMap &buffers = request->buffers();\n> > >>>> + for (const auto &[stream, buffer] : buffers) {\n> > >>>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n> > >>>> + if (in.isValid()) {\n> > >>>> + auto data = in.planes()[0].data();\n> > >>>> + auto streamConfig = stream->configuration();\n> > >>>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n> > >>>> +\n> > >>>> + std::function calcPixelMean;\n> > >>>> + int pixelStride;\n> > >>>> +\n> > >>>> + switch (streamConfig.pixelFormat) {\n> > >>>> + case formats::NV12:\n> > >>>> + calcPixelMean = calcPixelMeanNV12;\n> > >>>> + pixelStride = 1;\n> > >>>> + break;\n> > >>>> + case formats::SRGGB10:\n> > >>>> + calcPixelMean = calcPixelMeanRAW10;\n> > >>>> + pixelStride = 2;\n> > >>>> + break;\n> > >>>> + default:\n> > >>>> + std::stringstream s;\n> > >>>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n> > >>>> + throw std::invalid_argument(s.str());\n> > >>>> + }\n> > >>>> +\n> > >>>> + double sum = 0;\n> > >>>> + int w = 20;\n> > >>>> + int xs = streamConfig.size.width / 2 - w / 2;\n> > >>>> + int ys = streamConfig.size.height / 2 - w / 2;\n> > >>>> +\n> > >>>> + for (auto y = ys; y < ys + w; y++) {\n> > >>>> + auto line = data + y * streamConfig.stride;\n> > >>>> + for (auto x = xs; x < xs + w; x++) {\n> > >>>> + sum += calcPixelMean(line + x * pixelStride);\n> > >>>> + }\n> > >>>> + }\n> > >>>> + sum = sum / (w * w);\n> > >>>> + return sum;\n> > >>>> + }\n> > >>>> + }\n> > >>>> + return 0;\n> > >>>> +}\n> > >>>> +\n> > >>>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n> > >>>> + : controls_(idmap)\n> > >>>> +{\n> > >>>> +}\n> > >>>> +\n> > >>>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n> > >>>> +{\n> > >>>> + metadata_ = request->metadata();\n> > >>>> +\n> > >>>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n> > >>>> + if (previous) {\n> > >>>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n> > >>>> + }\n> > >>>> + sequence_ = request->sequence();\n> > >>>> +}\n> > >>>> +\n> > >>>> +void TimeSheetEntry::printInfo()\n> > >>>> +{\n> > >>>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n> > >>>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n> > >>>> +\n> > >>>> + if (!metadata_.empty()) {\n> > >>>> + std::cout << \"Metadata:\" << std::endl;\n> > >>>> + auto idMap = metadata_.idMap();\n> > >>>> + assert(idMap);\n> > >>>> + for (const auto &[id, value] : metadata_) {\n> > >>>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n> > >>>> + }\n> > >>>> + }\n> > >>>> +}\n> > >>>> +\n> > >>>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n> > >>>> +{\n> > >>>> + auto &entry = entries_[pos];\n> > >>>> + if (!entry)\n> > >>>> + entry = std::make_shared(idmap_);\n> > >>>> + return *entry;\n> > >>>> +}\n> > >>>> +\n> > >>>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n> > >>>> +{\n> > >>>> + request->controls() = get(sequence).controls();\n> > >>>> +}\n> > >>>> +\n> > >>>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n> > >>>> +{\n> > >>>> + uint32_t sequence = request->sequence();\n> > >>>> + auto &entry = get(sequence);\n> > >>>> + TimeSheetEntry *previous = nullptr;\n> > >>>> + if (sequence >= 1) {\n> > >>>> + previous = entries_[sequence - 1].get();\n> > >>>> + }\n> > >>>> +\n> > >>>> + entry.handleCompleteRequest(request, previous);\n> > >>>> +}\n> > >>>> +\n> > >>>> +void TimeSheet::printAllInfos()\n> > >>>> +{\n> > >>>> + for (auto entry : entries_) {\n> > >>>> + if (entry)\n> > >>>> + entry->printInfo();\n> > >>>> + }\n> > >>>> +}\n> > >>>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n> > >>>> new file mode 100644\n> > >>>> index 00000000..c155763c\n> > >>>> --- /dev/null\n> > >>>> +++ b/src/apps/lc-compliance/time_sheet.h\n> > >>>> @@ -0,0 +1,53 @@\n> > >>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > >>>> +/*\n> > >>>> + * Copyright (C) 2024, Ideas on Board Oy\n> > >>>> + *\n> > >>>> + * time_sheet.h\n> > >>>> + */\n> > >>>> +\n> > >>>> +#pragma once\n> > >>>> +\n> > >>>> +#include \n> > >>>> +#include \n> > >>>> +\n> > >>>> +#include \n> > >>>> +\n> > >>>> +class TimeSheetEntry\n> > >>>> +{\n> > >>>> +public:\n> > >>>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n> > >>>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n> > >>>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n> > >>>> +\n> > >>>> + libcamera::ControlList &controls() { return controls_; };\n> > >>>> + libcamera::ControlList &metadata() { return metadata_; };\n> > >>>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n> > >>>> + void printInfo();\n> > >>>> + double getSpotBrightness() const { return spotBrightness_; };\n> > >>>> + double getBrightnessChange() const { return brightnessChange_; };\n> > >>>> +\n> > >>>> +private:\n> > >>>> + double spotBrightness_ = 0.0;\n> > >>>> + double brightnessChange_ = 0.0;\n> > >>>> + libcamera::ControlList controls_;\n> > >>>> + libcamera::ControlList metadata_;\n> > >>>> + uint32_t sequence_ = 0;\n> > >>>> +};\n> > >>>> +\n> > >>>> +class TimeSheet\n> > >>>> +{\n> > >>>> +public:\n> > >>>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n> > >>>> + : idmap_(idmap), entries_(count){};\n> > >>>> +\n> > >>>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n> > >>>> + void handleCompleteRequest(libcamera::Request *request);\n> > >>>> + void printAllInfos();\n> > >>>> +\n> > >>>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n> > >>>> + TimeSheetEntry &get(size_t pos);\n> > >>>> +\n> > >>>> +private:\n> > >>>> + const libcamera::ControlIdMap &idmap_;\n> > >>>> + std::vector> entries_;\n> > >>>> +};\n> > >>>> --\n> > >>>> 2.40.1\n> > >>>>\n> > >>\n> > >> --\n> > >> Regards,\n> > >>\n> > >> Stefan Klug\n> > >>\n> >\n> > --\n> > Regards,\n> >\n> > Stefan Klug\n> >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 565BBC326B\n\tfor ;\n\tMon, 4 Mar 2024 11:58:39 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 8CE8E62867;\n\tMon, 4 Mar 2024 12:58:38 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 51C30627FC\n\tfor ;\n\tMon, 4 Mar 2024 12:58:37 +0100 (CET)","from pendragon.ideasonboard.com\n\t(aztw-30-b2-v4wan-166917-cust845.vm26.cable.virginm.net\n\t[82.37.23.78])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id EE1E33374;\n\tMon, 4 Mar 2024 12:58:20 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"rlrCPgsZ\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709553501;\n\tbh=5BtIW860TtrNoRrx3QU2pS6Lk1kx1pqoeOL9ijPykv8=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=rlrCPgsZ1dcF+uxWV5RYUthBE3bmdZu/zRQgYvusB0m3Nd6tzeQrB1EoZnBm+F8az\n\tN8uBUB+xqZUdqUWUh35O+QWS7s5MtyJFZTKRQ2B+u3UTP3OjIGOk8LzwF3YPvpIInR\n\tjwhG5/6WhSqAqdJID6NM+tKjlz67wAJg7Gi2GykM=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t\n\t\n\t\n\t<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>\n\t","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","From":"Kieran Bingham ","To":"David Plowman ,\n\tStefan Klug ","Date":"Mon, 04 Mar 2024 11:58:34 +0000","Message-ID":"<170955351406.1676185.13619836915117809277@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":28825,"web_url":"https://patchwork.libcamera.org/comment/28825/","msgid":"<7e2f343d-6989-43e8-8c60-f8197f31278b@ideasonboard.com>","date":"2024-03-04T16:27:29","subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi David,\n\nAm 04.03.24 um 12:35 schrieb David Plowman:\n> Hi Stefan\n> \n> On Fri, 1 Mar 2024 at 13:22, Stefan Klug wrote:\n>>\n>> Hi David,\n>>\n>> Am 01.03.24 um 13:04 schrieb David Plowman:\n>>> Hi Stefan\n>>>\n>>> On Fri, 1 Mar 2024 at 11:22, Stefan Klug wrote:\n>>>>\n>>>> Hi David,\n>>>>\n>>>> thanks for your comment. This is the same message I sent before, but with proper\n>>>> linewrapping - sorry about that.\n>>>>\n>>>> Am 01.03.24 um 08:56 schrieb David Plowman:\n>>>>> Hi Stefan\n>>>>>\n>>>>> Thanks for posting this and re-starting the discussion of per-frame controls.\n>>>>>\n>>>>> Could you perhaps just summarise exactly what you mean by per-frame\n>>>>> controls, that is to say, what is being tested here?\n>>>>>\n>>>>> Eye-balling the code, I think I understood that:\n>>>>>\n>>>>> * You are setting controls in a specific request.\n>>>>> * When that request completes, you are expecting the controls to have\n>>>>> taken effect in the images that were returned with that request (but\n>>>>> not earlier).\n>>>>>\n>>>>> But I wasn't totally sure - have I understood that correctly?\n>>>>>\n>>>>> Also, do we know if any other pipeline handlers implement this\n>>>>> behaviour? Do folks think that all pipeline handlers should implement\n>>>>> this behaviour?\n>>>>>\n>>>>> Thanks again!\n>>>>> David\n>>>>>\n>>>> You are completely right. This needs a bit more context.\n>>>>\n>>>> It all started on my side with implementing metadata support in SimplePipeline\n>>>> handler. I soon hit some corner cases where I expected things to behave\n>>>> differently in cooperation with DelayedControls. So either I hit a bug in\n>>>> DelayedControls or I didn't fully understand the concept behind it.\n>>>>\n>>>> I ended up changing several things in DelayedControls which I believe where\n>>>> correct. The question then was how to prove correctness as all current users of\n>>>> DelayedControls where ISP implementations. There are some unittests, but the\n>>>> order of calls in these tests also felt counterintuitive (might well be, that\n>>>> it's just missing knowledge on my side).\n>>>>\n>>>> In that area there were also no tests in lc-compliance which tested the\n>>>> behaviour of an actual sensor/isp combination. So I started to write these tests\n>>>> with my personal expectation of how I believe things should work. These tests\n>>>> pass on my SimplePipeline. I also tried to massage the rkisp pipeline to pass\n>>>> these tests and hit some corners where work would be required.\n>>>>\n>>>> Before digging deeper into that I believe it makes sense to see if my\n>>>> expectations are correct and if a broader audience agrees to them. So here we\n>>>> are, that's the reason for the RFC.\n>>>>\n>>>> Now on to the technical details:\n>>>> Yes my expectation on per-frame-controls would be: If I queue something for\n>>>> frame x, I expect that the system tries to fullfill exacty that request (and\n>>>> never earlier). I'm well aware that this will not be possible in every case when\n>>>> an ISP is involved, so the ISP should do the best it can, but the metadata\n>>>> should reflect what was actually achieved. All the tests are currently for\n>>>> manual mode, so no regulation is involved (at least for the params I test).\n>>>>\n>>>> Do you agree with these assumptions? Looking forward to your opinion.\n>>>\n>>> Thanks for the clarification, and glad that I've understood!\n>>>\n>>> A bit of context from our side... We looked at and implemented a\n>>> per-frame controls solution for the Raspberry Pi back in summer 2022.\n>>> I would say there were a couple of differences:\n>>>\n>>> 1. We included a mechanism for reporting failure cases. The only real\n>>> failure case we have is failing to update camera exposure/gain\n>>> settings in time because there is quite a short window in which to\n>>> service the camera frame interrupts (especially when the system is\n>>> busy and the framerate is fast).\n>>\n>> That's interesting. Could you explain the mechanics you used for that\n>> feedback or point me to a patchset? Would be interesting to see which\n>> failure cases you modelled. How did you test these?\n> \n> The last time we did any work on this was last summer for a libcamera\n> F2F in Prague. I think this\n> https://github.com/naushir/libcamera/tree/pfc_update was the code we\n> were running at that time. Obviously things have moved on since then,\n> not least the existence of Pi 5, which will need handling as well.\n> \n> The way the feedback worked was that you got a sequence number every\n> time you submitted a request with controls in it (distinct from the\n> request sequence number), which we called the \"submit id\". Every\n> request that completed contained one of these sequence numbers\n> indicating which the most recent controls were that had been applied,\n> and this was known as the \"sync id\". So to ask \"does this request\n> implement the controls that I set in it when I queued it?\" you would\n> check \"requset->syncId == request->submitId\".\n> \n>>>\n>>> The mechanism allowed the application to check whether controls had\n>>> been applied as expected, or whether they'd been delayed to the next\n>>> frame. It told you whether they'd got \"merged\" with the next set of\n>>> controls, or whether all the controls are now running \"a frame late\".\n>>>\n>>> 2. We implemented our scheme in the same way as you've described,\n>>> where the request where you set the controls has the first images that\n>>> fulfill the request. We called this \"Android mode\" though tbh I'm not\n>>> sure whether or not this really is what Android does!!!\n>>>\n>>> Although in the end we decided we didn't like this behaviour so much\n>>> because controls get delayed to the back of all the requests that have\n>>> been queued (and we normally queue quite a lot so as to avoid the risk\n>>\n>> Why was it necessary to move them to the back of all request? I guess in\n>> a typical usecase most requests would not contain controls (at least not\n>> from user side). So there would be a way to apply them as early in the\n>> queue as possible. I guess I'm missing somethimng here.\n> \n> Sorry, I'm not being super clear! We never \"moved\" controls to the\n> back of the requests, the problem is that when you queue a request\n> with some controls in it, those controls are necessarily behind all\n> the other requests that you queued previously. So you have potentially\n> many extra frames of latency.\n> \n> But you're exactly right, most requests don't contain any controls, so\n> we used to move the controls up the request queue so that they would\n> happen earlier.\n\nNow I got it! Thanks. That makes perfect sense now.\n\n> \n>>\n>>> of frame drops). So we also added \"Raspberry Pi mode\" where controls\n>>> are applied earlier.\n>>>\n>>> In principle I would be happy to let the Pi run in either mode\n>>> (\"Android\" or \"Pi\"), so long as we can choose \"Pi mode\" for our\n>>> applications. In both cases the same reporting mechanism told the\n>>> application exactly when the controls had been applied, so that\n>>> applications could be \"mode\" agnostic.\n>>>\n>>> Since Summer 2022, per-frame controls have not really progressed so\n>>> far as I know. We are intending to resurrect our per-frame controls\n>>> implementation again once we've got over the Pi 5 libcamera\n>>> integration, so I'm very happy to be resuming this discussion.\n>>\n>> Sure. Would be great if we could come up with a set of tests that wee\n>> all agree on.\n> \n> I agree! Though the last 18 months suggests to me that we're not there yet.\n> \n> There's currently work going on that splits buffers out of requests.\n> To me, the problem with the control lists suggests that possibly\n> controls should have their own queues and be removed from requests as\n> well. At which point this all becomes an even bigger discussion.\n> \n> So I'm kind of sorry to be listing all the ways in which per-frame\n> controls are an unresolved problem, but not really knowing where it\n> goes from here. Nonetheless, we are wanting to move forward once the\n> Pi 5 integration is out of the way.\n\nDon't feel sorry. At least I'm aware of some of the nasty details now :-)\n\nStefan\n\n> \n> David\n> \n>>\n>> Cheers,\n>> Stefan\n>>\n>>>\n>>> Thanks!\n>>> David\n>>>\n>>>>\n>>>> Best regards,\n>>>> Stefan\n>>>>\n>>>>>\n>>>>> On Thu, 29 Feb 2024 at 17:01, Stefan Klug wrote:\n>>>>>>\n>>>>>> These tests check if controls (only exposure time and analogue gain at\n>>>>>> the moment) get applied on the frame they were requested for.\n>>>>>>\n>>>>>> This is tested by looking at the metadata and additionally by\n>>>>>> calculating a mean brightness on a centered rect of 20x20 pixels.\n>>>>>>\n>>>>>> Until today, these tests where only run on a project specific branch\n>>>>>> with a modified simple pipeline. In theory they should pass on a\n>>>>>> current master :-)\n>>>>>>\n>>>>>> Current test setup: imx219 with simple pipeline on an imx8mp.\n>>>>>> Modifications of either the exposure delay or the gain delay in\n>>>>>> the camera_sensor class resulted in test failures.\n>>>>>> Which is exactly what this test shall proove.\n>>>>>>\n>>>>>> Signed-off-by: Stefan Klug \n>>>>>> ---\n>>>>>> src/apps/lc-compliance/capture_test.cpp | 39 ++++\n>>>>>> src/apps/lc-compliance/meson.build | 2 +\n>>>>>> src/apps/lc-compliance/per_frame_controls.cpp | 214 ++++++++++++++++++\n>>>>>> src/apps/lc-compliance/per_frame_controls.h | 41 ++++\n>>>>>> src/apps/lc-compliance/simple_capture.cpp | 4 +-\n>>>>>> src/apps/lc-compliance/simple_capture.h | 2 +-\n>>>>>> src/apps/lc-compliance/time_sheet.cpp | 135 +++++++++++\n>>>>>> src/apps/lc-compliance/time_sheet.h | 53 +++++\n>>>>>> 8 files changed, 487 insertions(+), 3 deletions(-)\n>>>>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.cpp\n>>>>>> create mode 100644 src/apps/lc-compliance/per_frame_controls.h\n>>>>>> create mode 100644 src/apps/lc-compliance/time_sheet.cpp\n>>>>>> create mode 100644 src/apps/lc-compliance/time_sheet.h\n>>>>>>\n>>>>>> diff --git a/src/apps/lc-compliance/capture_test.cpp b/src/apps/lc-compliance/capture_test.cpp\n>>>>>> index 1dcfcf92..43fe59f3 100644\n>>>>>> --- a/src/apps/lc-compliance/capture_test.cpp\n>>>>>> +++ b/src/apps/lc-compliance/capture_test.cpp\n>>>>>> @@ -11,6 +11,7 @@\n>>>>>> #include \n>>>>>>\n>>>>>> #include \"environment.h\"\n>>>>>> +#include \"per_frame_controls.h\"\n>>>>>> #include \"simple_capture.h\"\n>>>>>>\n>>>>>> using namespace libcamera;\n>>>>>> @@ -133,3 +134,41 @@ INSTANTIATE_TEST_SUITE_P(CaptureTests,\n>>>>>> testing::Combine(testing::ValuesIn(ROLES),\n>>>>>> testing::ValuesIn(NUMREQUESTS)),\n>>>>>> SingleStream::nameParameters);\n>>>>>> +\n>>>>>> +/*\n>>>>>> + * Test Per frame controls\n>>>>>> + */\n>>>>>> +TEST_F(SingleStream, testFramePreciseExposureChange)\n>>>>>> +{\n>>>>>> + PerFrameControls capture(camera_);\n>>>>>> + capture.configure(StreamRole::Viewfinder);\n>>>>>> + capture.testFramePreciseExposureChange();\n>>>>>> +}\n>>>>>> +\n>>>>>> +TEST_F(SingleStream, testFramePreciseGainChange)\n>>>>>> +{\n>>>>>> + PerFrameControls capture(camera_);\n>>>>>> + capture.configure(StreamRole::Viewfinder);\n>>>>>> + capture.testFramePreciseGainChange();\n>>>>>> +}\n>>>>>> +\n>>>>>> +TEST_F(SingleStream, testExposureGainIsAppliedOnFirstFrame)\n>>>>>> +{\n>>>>>> + PerFrameControls capture(camera_);\n>>>>>> + capture.configure(StreamRole::Viewfinder);\n>>>>>> + capture.testExposureGainIsAppliedOnFirstFrame();\n>>>>>> +}\n>>>>>> +\n>>>>>> +TEST_F(SingleStream, testExposureGainFromFirstRequestGetsApplied)\n>>>>>> +{\n>>>>>> + PerFrameControls capture(camera_);\n>>>>>> + capture.configure(StreamRole::Viewfinder);\n>>>>>> + capture.testExposureGainFromFirstRequestGetsApplied();\n>>>>>> +}\n>>>>>> +\n>>>>>> +TEST_F(SingleStream, testExposureGainFromFirstAndSecondRequestGetsApplied)\n>>>>>> +{\n>>>>>> + PerFrameControls capture(camera_);\n>>>>>> + capture.configure(StreamRole::Viewfinder);\n>>>>>> + capture.testExposureGainFromFirstAndSecondRequestGetsApplied();\n>>>>>> +}\n>>>>>> diff --git a/src/apps/lc-compliance/meson.build b/src/apps/lc-compliance/meson.build\n>>>>>> index c792f072..2a6f52af 100644\n>>>>>> --- a/src/apps/lc-compliance/meson.build\n>>>>>> +++ b/src/apps/lc-compliance/meson.build\n>>>>>> @@ -15,7 +15,9 @@ lc_compliance_sources = files([\n>>>>>> 'capture_test.cpp',\n>>>>>> 'environment.cpp',\n>>>>>> 'main.cpp',\n>>>>>> + 'per_frame_controls.cpp',\n>>>>>> 'simple_capture.cpp',\n>>>>>> + 'time_sheet.cpp',\n>>>>>> ])\n>>>>>>\n>>>>>> lc_compliance = executable('lc-compliance', lc_compliance_sources,\n>>>>>> diff --git a/src/apps/lc-compliance/per_frame_controls.cpp b/src/apps/lc-compliance/per_frame_controls.cpp\n>>>>>> new file mode 100644\n>>>>>> index 00000000..70fc44ac\n>>>>>> --- /dev/null\n>>>>>> +++ b/src/apps/lc-compliance/per_frame_controls.cpp\n>>>>>> @@ -0,0 +1,214 @@\n>>>>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>>>>>> +/*\n>>>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>>>> + *\n>>>>>> + * per_frame_controls.cpp - Tests for per frame controls\n>>>>>> + */\n>>>>>> +#include \"per_frame_controls.h\"\n>>>>>> +\n>>>>>> +#include \n>>>>>> +\n>>>>>> +#include \"time_sheet.h\"\n>>>>>> +\n>>>>>> +using namespace libcamera;\n>>>>>> +\n>>>>>> +PerFrameControls::PerFrameControls(std::shared_ptr camera)\n>>>>>> + : SimpleCapture(camera)\n>>>>>> +{\n>>>>>> +}\n>>>>>> +\n>>>>>> +std::shared_ptr PerFrameControls::startCaptureWithTimeSheet(unsigned int framesToCapture, const ControlList *controls)\n>>>>>> +{\n>>>>>> + ControlList ctrls(camera_->controls().idmap());\n>>>>>> + /* Ensure defined default values */\n>>>>>> + ctrls.set(controls::AeEnable, false);\n>>>>>> + ctrls.set(controls::AeExposureMode, controls::ExposureCustom);\n>>>>>> + ctrls.set(controls::ExposureTime, 10000);\n>>>>>> + ctrls.set(controls::AnalogueGain, 1.0);\n>>>>>> +\n>>>>>> + if (controls) {\n>>>>>> + ctrls.merge(*controls, true);\n>>>>>> + }\n>>>>>> +\n>>>>>> + start(&ctrls);\n>>>>>> +\n>>>>>> + queueCount_ = 0;\n>>>>>> + captureCount_ = 0;\n>>>>>> + captureLimit_ = framesToCapture;\n>>>>>> +\n>>>>>> + auto timeSheet = std::make_shared(captureLimit_, camera_->controls().idmap());\n>>>>>> + timeSheet_ = timeSheet;\n>>>>>> + return timeSheet;\n>>>>>> +}\n>>>>>> +\n>>>>>> +int PerFrameControls::queueRequest(Request *request)\n>>>>>> +{\n>>>>>> + queueCount_++;\n>>>>>> + if (queueCount_ > captureLimit_)\n>>>>>> + return 0;\n>>>>>> +\n>>>>>> + auto ts = timeSheet_.lock();\n>>>>>> + if (ts) {\n>>>>>> + ts->prepareForQueue(request, queueCount_ - 1);\n>>>>>> + }\n>>>>>> +\n>>>>>> + return camera_->queueRequest(request);\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::requestComplete(Request *request)\n>>>>>> +{\n>>>>>> + auto ts = timeSheet_.lock();\n>>>>>> + if (ts) {\n>>>>>> + ts->handleCompleteRequest(request);\n>>>>>> + }\n>>>>>> +\n>>>>>> + captureCount_++;\n>>>>>> + if (captureCount_ >= captureLimit_) {\n>>>>>> + loop_->exit(0);\n>>>>>> + return;\n>>>>>> + }\n>>>>>> +\n>>>>>> + request->reuse(Request::ReuseBuffers);\n>>>>>> + if (queueRequest(request))\n>>>>>> + loop_->exit(-EINVAL);\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::runCaptureSession()\n>>>>>> +{\n>>>>>> + Stream *stream = config_->at(0).stream();\n>>>>>> + const std::vector> &buffers = allocator_->buffers(stream);\n>>>>>> +\n>>>>>> + /* Queue the recommended number of reqeuests. */\n>>>>>> + for (const std::unique_ptr &buffer : buffers) {\n>>>>>> + std::unique_ptr request = camera_->createRequest();\n>>>>>> + request->addBuffer(stream, buffer.get());\n>>>>>> + queueRequest(request.get());\n>>>>>> + requests_.push_back(std::move(request));\n>>>>>> + }\n>>>>>> +\n>>>>>> + /* Run capture session. */\n>>>>>> + loop_ = new EventLoop();\n>>>>>> + loop_->exec();\n>>>>>> + stop();\n>>>>>> + delete loop_;\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::testFramePreciseExposureChange()\n>>>>>> +{\n>>>>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>>>>>> + auto &ts = *timeSheet;\n>>>>>> +\n>>>>>> +\n>>>>>> + ts[3].controls().set(controls::ExposureTime, 5000);\n>>>>>> + //wait a few frames to settle\n>>>>>> + ts[6].controls().set(controls::ExposureTime, 20000);\n>>>>>> + ts.printAllInfos();\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + EXPECT_NEAR(ts[5].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>>>>>> + EXPECT_NEAR(ts[6].metadata().get(controls::ExposureTime).value(), 20000, 20);\n>>>>>> +\n>>>>>> + /* No increase just before setting exposure */\n>>>>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>>>>>> + /*\n>>>>>> + * Todo: The change is brightness was a bit low (Exposure time increase by 4x resulted in a brightness increase of < 2).\n>>>>>> + * This should be investigated.\n>>>>>> + */\n>>>>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.3) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>>>>>> + << ts[3].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>>>>>> +\n>>>>>> + /* No increase just after setting exposure */\n>>>>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>>>>>> +\n>>>>>> + /* No increase just after setting exposure */\n>>>>>> + EXPECT_NEAR(ts[8].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much2 frames after the expected time of change (control delay too low?).\";\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::testFramePreciseGainChange()\n>>>>>> +{\n>>>>>> + auto timeSheet = startCaptureWithTimeSheet(10);\n>>>>>> + auto &ts = *timeSheet;\n>>>>>> +\n>>>>>> + ts[3].controls().set(controls::AnalogueGain, 1.0);\n>>>>>> + //wait a few frames to settle\n>>>>>> + ts[6].controls().set(controls::AnalogueGain, 4.0);\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + EXPECT_NEAR(ts[5].metadata().get(controls::AnalogueGain).value(), 1.0, 0.1);\n>>>>>> + EXPECT_NEAR(ts[6].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>>>> +\n>>>>>> + /* No increase just before setting gain */\n>>>>>> + EXPECT_NEAR(ts[5].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much before the expected time of change (control delay too high?).\";\n>>>>>> + /*\n>>>>>> + * Todo: I see a brightness change of roughly half the expected one. This is not yet understood and needs investigation\n>>>>>> + */\n>>>>>> + EXPECT_GT(ts[6].getBrightnessChange(), 1.7) << \"Brightness in frame \" << 6 << \" did not increase as expected (reference: \"\n>>>>>> + << ts[5].getSpotBrightness() << \" current: \" << ts[6].getSpotBrightness() << \" )\" << std::endl;\n>>>>>> +\n>>>>>> + /* No increase just after setting gain */\n>>>>>> + EXPECT_NEAR(ts[7].getBrightnessChange(), 1.0, 0.05) << \"Brightness changed too much after the expected time of change (control delay too low?).\";\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::testExposureGainFromFirstRequestGetsApplied()\n>>>>>> +{\n>>>>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>>>>>> + auto &ts = *timeSheet;\n>>>>>> +\n>>>>>> + ts[0].controls().set(controls::ExposureTime, 10000);\n>>>>>> + ts[0].controls().set(controls::AnalogueGain, 4.0);\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + /* We expect it to be applied after 3 frames, the latest*/\n>>>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>>>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::testExposureGainFromFirstAndSecondRequestGetsApplied()\n>>>>>> +{\n>>>>>> + auto timeSheet = startCaptureWithTimeSheet(5);\n>>>>>> + auto &ts = *timeSheet;\n>>>>>> +\n>>>>>> + ts[0].controls().set(controls::ExposureTime, 8000);\n>>>>>> + ts[0].controls().set(controls::AnalogueGain, 2.0);\n>>>>>> + ts[1].controls().set(controls::ExposureTime, 10000);\n>>>>>> + ts[1].controls().set(controls::AnalogueGain, 4.0);\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + /* We expect it to be applied after 3 frames, the latest*/\n>>>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::ExposureTime).value(), 10000, 20);\n>>>>>> + EXPECT_NEAR(ts[4].metadata().get(controls::AnalogueGain).value(), 4.0, 0.1);\n>>>>>> +}\n>>>>>> +\n>>>>>> +void PerFrameControls::testExposureGainIsAppliedOnFirstFrame()\n>>>>>> +{\n>>>>>> + ControlList startValues;\n>>>>>> + startValues.set(controls::ExposureTime, 5000);\n>>>>>> + startValues.set(controls::AnalogueGain, 1.0);\n>>>>>> +\n>>>>>> + auto ts1 = startCaptureWithTimeSheet(3, &startValues);\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::ExposureTime).value(), 5000, 20);\n>>>>>> + EXPECT_NEAR((*ts1)[0].metadata().get(controls::AnalogueGain).value(), 1.0, 0.01);\n>>>>>> +\n>>>>>> + /* Second capture with different values to ensure we don't hit default/old values */\n>>>>>> +\n>>>>>> + startValues.set(controls::ExposureTime, 15000);\n>>>>>> + startValues.set(controls::AnalogueGain, 4.0);\n>>>>>> +\n>>>>>> + auto ts2 = startCaptureWithTimeSheet(3, &startValues);\n>>>>>> +\n>>>>>> + runCaptureSession();\n>>>>>> +\n>>>>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::ExposureTime).value(), 15000, 20);\n>>>>>> + EXPECT_NEAR((*ts2)[0].metadata().get(controls::AnalogueGain).value(), 4.0, 0.01);\n>>>>>> +\n>>>>>> + /* with 3x exposure and 4x gain we could expect a brightness increase of 2x */\n>>>>>> + double brightnessChange = ts2->get(1).getSpotBrightness() / ts1->get(1).getSpotBrightness();\n>>>>>> + EXPECT_GT(brightnessChange, 2.0);\n>>>>>> +}\n>>>>>> diff --git a/src/apps/lc-compliance/per_frame_controls.h b/src/apps/lc-compliance/per_frame_controls.h\n>>>>>> new file mode 100644\n>>>>>> index 00000000..e783f024\n>>>>>> --- /dev/null\n>>>>>> +++ b/src/apps/lc-compliance/per_frame_controls.h\n>>>>>> @@ -0,0 +1,41 @@\n>>>>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n>>>>>> +/*\n>>>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>>>> + *\n>>>>>> + * per_frame_controls.h - Tests for per frame controls\n>>>>>> + */\n>>>>>> +\n>>>>>> +#pragma once\n>>>>>> +\n>>>>>> +#include \n>>>>>> +\n>>>>>> +#include \n>>>>>> +\n>>>>>> +#include \"../common/event_loop.h\"\n>>>>>> +\n>>>>>> +#include \"simple_capture.h\"\n>>>>>> +#include \"time_sheet.h\"\n>>>>>> +\n>>>>>> +class PerFrameControls : public SimpleCapture\n>>>>>> +{\n>>>>>> +public:\n>>>>>> + PerFrameControls(std::shared_ptr camera);\n>>>>>> +\n>>>>>> + std::shared_ptr startCaptureWithTimeSheet(unsigned int framesToCapture, const libcamera::ControlList *controls = nullptr);\n>>>>>> + void runCaptureSession();\n>>>>>> +\n>>>>>> + void testFramePreciseExposureChange();\n>>>>>> + void testFramePreciseGainChange();\n>>>>>> + void testExposureGainIsAppliedOnFirstFrame();\n>>>>>> + void testExposureGainFromFirstRequestGetsApplied();\n>>>>>> + void testExposureGainFromFirstAndSecondRequestGetsApplied();\n>>>>>> +\n>>>>>> + int queueRequest(libcamera::Request *request);\n>>>>>> + void requestComplete(libcamera::Request *request) override;\n>>>>>> +\n>>>>>> + unsigned int queueCount_;\n>>>>>> + unsigned int captureCount_;\n>>>>>> + unsigned int captureLimit_;\n>>>>>> +\n>>>>>> + std::weak_ptr timeSheet_;\n>>>>>> +};\n>>>>>> diff --git a/src/apps/lc-compliance/simple_capture.cpp b/src/apps/lc-compliance/simple_capture.cpp\n>>>>>> index cf4d7cf3..56680a83 100644\n>>>>>> --- a/src/apps/lc-compliance/simple_capture.cpp\n>>>>>> +++ b/src/apps/lc-compliance/simple_capture.cpp\n>>>>>> @@ -42,7 +42,7 @@ void SimpleCapture::configure(StreamRole role)\n>>>>>> }\n>>>>>> }\n>>>>>>\n>>>>>> -void SimpleCapture::start()\n>>>>>> +void SimpleCapture::start(const ControlList *controls)\n>>>>>> {\n>>>>>> Stream *stream = config_->at(0).stream();\n>>>>>> int count = allocator_->allocate(stream);\n>>>>>> @@ -52,7 +52,7 @@ void SimpleCapture::start()\n>>>>>>\n>>>>>> camera_->requestCompleted.connect(this, &SimpleCapture::requestComplete);\n>>>>>>\n>>>>>> - ASSERT_EQ(camera_->start(), 0) << \"Failed to start camera\";\n>>>>>> + ASSERT_EQ(camera_->start(controls), 0) << \"Failed to start camera\";\n>>>>>> }\n>>>>>>\n>>>>>> void SimpleCapture::stop()\n>>>>>> diff --git a/src/apps/lc-compliance/simple_capture.h b/src/apps/lc-compliance/simple_capture.h\n>>>>>> index 2911d601..54b1d54b 100644\n>>>>>> --- a/src/apps/lc-compliance/simple_capture.h\n>>>>>> +++ b/src/apps/lc-compliance/simple_capture.h\n>>>>>> @@ -22,7 +22,7 @@ protected:\n>>>>>> SimpleCapture(std::shared_ptr camera);\n>>>>>> virtual ~SimpleCapture();\n>>>>>>\n>>>>>> - void start();\n>>>>>> + void start(const libcamera::ControlList *controls = nullptr);\n>>>>>> void stop();\n>>>>>>\n>>>>>> virtual void requestComplete(libcamera::Request *request) = 0;\n>>>>>> diff --git a/src/apps/lc-compliance/time_sheet.cpp b/src/apps/lc-compliance/time_sheet.cpp\n>>>>>> new file mode 100644\n>>>>>> index 00000000..9a0e6544\n>>>>>> --- /dev/null\n>>>>>> +++ b/src/apps/lc-compliance/time_sheet.cpp\n>>>>>> @@ -0,0 +1,135 @@\n>>>>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>>>>>> +/*\n>>>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>>>> + *\n>>>>>> + * time_sheet.cpp\n>>>>>> + */\n>>>>>> +#include \"time_sheet.h\"\n>>>>>> +\n>>>>>> +#include \n>>>>>> +#include \n>>>>>> +\n>>>>>> +#include \"libcamera/internal/formats.h\"\n>>>>>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>>>>>> +\n>>>>>> +using namespace libcamera;\n>>>>>> +\n>>>>>> +double calcPixelMeanNV12(const uint8_t *data)\n>>>>>> +{\n>>>>>> + return (double)*data;\n>>>>>> +}\n>>>>>> +\n>>>>>> +double calcPixelMeanRAW10(const uint8_t *data)\n>>>>>> +{\n>>>>>> + return (double)*((const uint16_t *)data);\n>>>>>> +}\n>>>>>> +\n>>>>>> +double calculateMeanBrightnessFromCenterSpot(libcamera::Request *request)\n>>>>>> +{\n>>>>>> + const Request::BufferMap &buffers = request->buffers();\n>>>>>> + for (const auto &[stream, buffer] : buffers) {\n>>>>>> + MappedFrameBuffer in(buffer, MappedFrameBuffer::MapFlag::Read);\n>>>>>> + if (in.isValid()) {\n>>>>>> + auto data = in.planes()[0].data();\n>>>>>> + auto streamConfig = stream->configuration();\n>>>>>> + auto formatInfo = PixelFormatInfo::info(streamConfig.pixelFormat);\n>>>>>> +\n>>>>>> + std::function calcPixelMean;\n>>>>>> + int pixelStride;\n>>>>>> +\n>>>>>> + switch (streamConfig.pixelFormat) {\n>>>>>> + case formats::NV12:\n>>>>>> + calcPixelMean = calcPixelMeanNV12;\n>>>>>> + pixelStride = 1;\n>>>>>> + break;\n>>>>>> + case formats::SRGGB10:\n>>>>>> + calcPixelMean = calcPixelMeanRAW10;\n>>>>>> + pixelStride = 2;\n>>>>>> + break;\n>>>>>> + default:\n>>>>>> + std::stringstream s;\n>>>>>> + s << \"Unsupported Pixelformat \" << formatInfo.name;\n>>>>>> + throw std::invalid_argument(s.str());\n>>>>>> + }\n>>>>>> +\n>>>>>> + double sum = 0;\n>>>>>> + int w = 20;\n>>>>>> + int xs = streamConfig.size.width / 2 - w / 2;\n>>>>>> + int ys = streamConfig.size.height / 2 - w / 2;\n>>>>>> +\n>>>>>> + for (auto y = ys; y < ys + w; y++) {\n>>>>>> + auto line = data + y * streamConfig.stride;\n>>>>>> + for (auto x = xs; x < xs + w; x++) {\n>>>>>> + sum += calcPixelMean(line + x * pixelStride);\n>>>>>> + }\n>>>>>> + }\n>>>>>> + sum = sum / (w * w);\n>>>>>> + return sum;\n>>>>>> + }\n>>>>>> + }\n>>>>>> + return 0;\n>>>>>> +}\n>>>>>> +\n>>>>>> +TimeSheetEntry::TimeSheetEntry(const ControlIdMap &idmap)\n>>>>>> + : controls_(idmap)\n>>>>>> +{\n>>>>>> +}\n>>>>>> +\n>>>>>> +void TimeSheetEntry::handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous)\n>>>>>> +{\n>>>>>> + metadata_ = request->metadata();\n>>>>>> +\n>>>>>> + spotBrightness_ = calculateMeanBrightnessFromCenterSpot(request);\n>>>>>> + if (previous) {\n>>>>>> + brightnessChange_ = spotBrightness_ / previous->getSpotBrightness();\n>>>>>> + }\n>>>>>> + sequence_ = request->sequence();\n>>>>>> +}\n>>>>>> +\n>>>>>> +void TimeSheetEntry::printInfo()\n>>>>>> +{\n>>>>>> + std::cout << \"=== Frame \" << sequence_ << std::endl;\n>>>>>> + std::cout << \"Brightness: \" << spotBrightness_ << std::endl;\n>>>>>> +\n>>>>>> + if (!metadata_.empty()) {\n>>>>>> + std::cout << \"Metadata:\" << std::endl;\n>>>>>> + auto idMap = metadata_.idMap();\n>>>>>> + assert(idMap);\n>>>>>> + for (const auto &[id, value] : metadata_) {\n>>>>>> + std::cout << \" \" << idMap->at(id)->name() << \" : \" << value.toString() << std::endl;\n>>>>>> + }\n>>>>>> + }\n>>>>>> +}\n>>>>>> +\n>>>>>> +TimeSheetEntry &TimeSheet::get(size_t pos)\n>>>>>> +{\n>>>>>> + auto &entry = entries_[pos];\n>>>>>> + if (!entry)\n>>>>>> + entry = std::make_shared(idmap_);\n>>>>>> + return *entry;\n>>>>>> +}\n>>>>>> +\n>>>>>> +void TimeSheet::prepareForQueue(libcamera::Request *request, uint32_t sequence)\n>>>>>> +{\n>>>>>> + request->controls() = get(sequence).controls();\n>>>>>> +}\n>>>>>> +\n>>>>>> +void TimeSheet::handleCompleteRequest(libcamera::Request *request)\n>>>>>> +{\n>>>>>> + uint32_t sequence = request->sequence();\n>>>>>> + auto &entry = get(sequence);\n>>>>>> + TimeSheetEntry *previous = nullptr;\n>>>>>> + if (sequence >= 1) {\n>>>>>> + previous = entries_[sequence - 1].get();\n>>>>>> + }\n>>>>>> +\n>>>>>> + entry.handleCompleteRequest(request, previous);\n>>>>>> +}\n>>>>>> +\n>>>>>> +void TimeSheet::printAllInfos()\n>>>>>> +{\n>>>>>> + for (auto entry : entries_) {\n>>>>>> + if (entry)\n>>>>>> + entry->printInfo();\n>>>>>> + }\n>>>>>> +}\n>>>>>> diff --git a/src/apps/lc-compliance/time_sheet.h b/src/apps/lc-compliance/time_sheet.h\n>>>>>> new file mode 100644\n>>>>>> index 00000000..c155763c\n>>>>>> --- /dev/null\n>>>>>> +++ b/src/apps/lc-compliance/time_sheet.h\n>>>>>> @@ -0,0 +1,53 @@\n>>>>>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>>>>>> +/*\n>>>>>> + * Copyright (C) 2024, Ideas on Board Oy\n>>>>>> + *\n>>>>>> + * time_sheet.h\n>>>>>> + */\n>>>>>> +\n>>>>>> +#pragma once\n>>>>>> +\n>>>>>> +#include \n>>>>>> +#include \n>>>>>> +\n>>>>>> +#include \n>>>>>> +\n>>>>>> +class TimeSheetEntry\n>>>>>> +{\n>>>>>> +public:\n>>>>>> + TimeSheetEntry(const libcamera::ControlIdMap &idmap);\n>>>>>> + TimeSheetEntry(TimeSheetEntry &&other) noexcept = default;\n>>>>>> + TimeSheetEntry(const TimeSheetEntry &) = delete;\n>>>>>> +\n>>>>>> + libcamera::ControlList &controls() { return controls_; };\n>>>>>> + libcamera::ControlList &metadata() { return metadata_; };\n>>>>>> + void handleCompleteRequest(libcamera::Request *request, const TimeSheetEntry *previous);\n>>>>>> + void printInfo();\n>>>>>> + double getSpotBrightness() const { return spotBrightness_; };\n>>>>>> + double getBrightnessChange() const { return brightnessChange_; };\n>>>>>> +\n>>>>>> +private:\n>>>>>> + double spotBrightness_ = 0.0;\n>>>>>> + double brightnessChange_ = 0.0;\n>>>>>> + libcamera::ControlList controls_;\n>>>>>> + libcamera::ControlList metadata_;\n>>>>>> + uint32_t sequence_ = 0;\n>>>>>> +};\n>>>>>> +\n>>>>>> +class TimeSheet\n>>>>>> +{\n>>>>>> +public:\n>>>>>> + TimeSheet(int count, const libcamera::ControlIdMap &idmap)\n>>>>>> + : idmap_(idmap), entries_(count){};\n>>>>>> +\n>>>>>> + void prepareForQueue(libcamera::Request *request, uint32_t sequence);\n>>>>>> + void handleCompleteRequest(libcamera::Request *request);\n>>>>>> + void printAllInfos();\n>>>>>> +\n>>>>>> + TimeSheetEntry &operator[](size_t pos) { return get(pos); };\n>>>>>> + TimeSheetEntry &get(size_t pos);\n>>>>>> +\n>>>>>> +private:\n>>>>>> + const libcamera::ControlIdMap &idmap_;\n>>>>>> + std::vector> entries_;\n>>>>>> +};\n>>>>>> --\n>>>>>> 2.40.1\n>>>>>>\n>>>>\n>>>> --\n>>>> Regards,\n>>>>\n>>>> Stefan Klug\n>>>>\n>>\n>> --\n>> Regards,\n>>\n>> Stefan Klug\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id ABDA2C326B\n\tfor ;\n\tMon, 4 Mar 2024 16:27:35 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1FD7662867;\n\tMon, 4 Mar 2024 17:27:35 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 6AA77627FC\n\tfor ;\n\tMon, 4 Mar 2024 17:27:33 +0100 (CET)","from [IPV6:2a00:6020:448c:6c00:560e:6da1:ef9c:32d0] (unknown\n\t[IPv6:2a00:6020:448c:6c00:560e:6da1:ef9c:32d0])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id CE0EE3871;\n\tMon, 4 Mar 2024 17:27:16 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"CTr+linB\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1709569637;\n\tbh=yc0Rr9B7wK2PZPkxLLEN/Az3wJwCBsMuUqUBPFsZqbc=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=CTr+linBCkYYdriehQzEIYt7VYGDP0ONB8vpQZE3RJL6PN9cWOAsXglJz/oVmgY9z\n\t9fiyX9cLZ3a1qPALCTFl5tEGhbJ2C4rMjDJfPVfVqjx6Yfysn7GXggajTsYIY6eFkB\n\tbd7U3iLQ88rlzW0pmgX6l15qWEkC0avnx2IYq4P0=","Message-ID":"<7e2f343d-6989-43e8-8c60-f8197f31278b@ideasonboard.com>","Date":"Mon, 4 Mar 2024 17:27:29 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [RFC] libcamera: lc-compliance: Add initial set of per frame\n\tcontrol tests","Content-Language":"en-US","To":"David Plowman ","References":"<20240229170115.159450-1-stefan.klug@ideasonboard.com>\n\t\n\t\n\t\n\t<5828a270-3b25-453f-b234-856f5e69f16d@ideasonboard.com>\n\t","From":"Stefan Klug ","In-Reply-To":"","Content-Type":"text/plain; charset=UTF-8","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]