[{"id":27825,"web_url":"https://patchwork.libcamera.org/comment/27825/","msgid":"","date":"2023-09-21T10:40:31","subject":"Re: [libcamera-devel] [RFC PATCH v2 3/4] libcamera: converter: add\n\tsoftware converter","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Andrey\n\nSw converters will be discussed in detail, so I'll only go through the\ntrivial/cosmetics things and leave discussion on the architecture for\nthe face-2-face meeting in Paris\n\nOn Wed, Sep 20, 2023 at 06:19:20PM +0300, Andrey Konovalov via libcamera-devel wrote:\n> Use the Converter interface to implement bilinear Bayer demosaic\n> filtering and very simple AWB on CPU.\n>\n> The only output format currently supported is RGB888.\n>\n> Signed-off-by: Andrey Konovalov \n> ---\n> .../internal/converter/converter_softw.h | 90 ++++\n> .../libcamera/internal/converter/meson.build | 1 +\n> src/libcamera/converter/converter_softw.cpp | 410 ++++++++++++++++++\n> src/libcamera/converter/meson.build | 3 +-\n> 4 files changed, 503 insertions(+), 1 deletion(-)\n> create mode 100644 include/libcamera/internal/converter/converter_softw.h\n> create mode 100644 src/libcamera/converter/converter_softw.cpp\n>\n> diff --git a/include/libcamera/internal/converter/converter_softw.h b/include/libcamera/internal/converter/converter_softw.h\n> new file mode 100644\n> index 00000000..eff9d6b2\n> --- /dev/null\n> +++ b/include/libcamera/internal/converter/converter_softw.h\n> @@ -0,0 +1,90 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2023, Linaro Ltd\n> + *\n> + * converter_softw.h - interface of software converter (runs 100% on CPU)\n> + */\n> +\n\n#include \n\n> +#include \n\n#include \n#include \n\n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#include \"libcamera/internal/converter.h\"\n> +\n> +namespace libcamera {\n> +\n> +class FrameBuffer;\n> +class MediaDevice;\n> +class Size;\n> +class SizeRange;\n> +struct StreamConfiguration;\n> +\n> +class SwConverter : public Converter\n> +{\n> +public:\n> +\tSwConverter([[maybe_unused]] MediaDevice *media)\n\nThis is not super-nice, but I don't have anything better to suggest at\nthsi time unless we completely revisit the implementation of [2/4]. I\nguess we can live with this even if it might indicate we should be\ndoing better.\n\nI had a go at using SFINAE to selectively compile in the version of\nConverterFactory::createInstance() that passes in a MediaDevice\nselectively, but after 30 minutes I gave up :)\n\n\n> +\t\t: Converter() {}\n> +\n> +\tint loadConfiguration([[maybe_unused]] const std::string &filename) { return 0; }\n> +\tbool isValid() const { return true; }\n> +\n> +\tstd::vector formats(PixelFormat input);\n> +\tSizeRange sizes(const Size &input);\n> +\n> +\tstd::tuple\n> +\tstrideAndFrameSize(const PixelFormat &pixelFormat, const Size &size);\n> +\n> +\tint configure(const StreamConfiguration &inputCfg,\n> +\t\t const std::vector> &outputCfg);\n> +\tint exportBuffers(unsigned int ouput, unsigned int count,\n> +\t\t\t std::vector> *buffers);\n> +\n> +\tvoid process(FrameBuffer *input, FrameBuffer *output);\n> +\tint start();\n> +\tvoid stop();\n> +\n> +\tint queueBuffers(FrameBuffer *input,\n> +\t\t\t const std::map &outputs);\n> +\n> +private:\n> +\tclass Isp : public Object\n> +\t{\n> +\tpublic:\n> +\t\tIsp(SwConverter *converter)\n> +\t\t\t: converter_(converter) {}\n> +\n> +\t\tint configure(const StreamConfiguration &inputCfg,\n> +\t\t\t const StreamConfiguration &outputCfg);\n> +\t\tint exportBuffers(unsigned int count,\n> +\t\t\t\t std::vector> *buffers);\n> +\t\tvoid process(FrameBuffer *input, FrameBuffer *output);\n> +\t\tint start();\n> +\t\tvoid stop();\n> +\n> +\tprivate:\n> +\t\tvoid debayer(uint8_t *dst, const uint8_t *src);\n> +\n> +\t\tSwConverter *converter_;\n> +\n> +\t\tThread thread_;\n> +\n> +\t\tunsigned int width_;\n> +\t\tunsigned int height_;\n> +\t\tunsigned int stride_;\n> +\t\tPoint red_shift_;\n> +\n> +\t\tunsigned long rNumerat_, rDenomin_; /* red gain for AWB */\n> +\t\tunsigned long bNumerat_, bDenomin_; /* blue gain for AWB */\n> +\t\tunsigned long gNumerat_, gDenomin_; /* green gain for AWB */\n> +\t};\n> +\n> +\tstd::unique_ptr isp_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/include/libcamera/internal/converter/meson.build b/include/libcamera/internal/converter/meson.build\n> index 891e79e7..843a0483 100644\n> --- a/include/libcamera/internal/converter/meson.build\n> +++ b/include/libcamera/internal/converter/meson.build\n> @@ -1,5 +1,6 @@\n> # SPDX-License-Identifier: CC0-1.0\n>\n> libcamera_internal_headers += files([\n> + 'converter_softw.h',\n> 'converter_v4l2_m2m.h',\n> ])\n> diff --git a/src/libcamera/converter/converter_softw.cpp b/src/libcamera/converter/converter_softw.cpp\n> new file mode 100644\n> index 00000000..ad7e6f3d\n> --- /dev/null\n> +++ b/src/libcamera/converter/converter_softw.cpp\n> @@ -0,0 +1,410 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2023, Linaro Ltd\n> + *\n> + * converter_softw.h - interface of software converter (runs 100% on CPU)\n> + */\n> +\n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \"libcamera/internal/bayer_format.h\"\n> +#include \"libcamera/internal/framebuffer.h\"\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +#include \"libcamera/internal/converter/converter_softw.h\"\n\nInclude this first to make sure it's self contained\n\nAnd you can remove a few duplicated includes like \n\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(Converter)\n> +\n> +std::vector SwConverter::formats(PixelFormat input)\n> +{\n> +\tstd::vector pixelFormats;\n> +\tBayerFormat inputFormat = BayerFormat::fromPixelFormat(input);\n> +\n> +\t/* Only RAW10P is currently supported */\n\nDo you care about the bayer ordering or any permutation is fine\n(SRGGB10, SBGGR10 etc etc)\n\n> +\tif (inputFormat.bitDepth == 10 && inputFormat.packing == BayerFormat::Packing::CSI2)\n> +\t\tpixelFormats.push_back(formats::RGB888);\n> +\n> +\tif (pixelFormats.empty())\n> +\t\tLOG(Converter, Info)\n> +\t\t\t<< \"Unsupported input format \" << input.toString();\n> +\n> +\treturn pixelFormats;\n\nIsn't\n\tBayerFormat inputFormat = BayerFormat::fromPixelFormat(input);\n\tif (inputFormat.bitDepth != 10 ||\n\t inputFormat.packing != BayerFormat::Packing::CSI2) {\n\t\tLOG(Converter, Info)\n\t\t\t<< \"Unsupported input format \" << input.toString();\n return {};\n\t}\n\n return { formats::RGB888 };\n\na bit simpler ?\n\n\n> +}\n> +\n> +SizeRange SwConverter::sizes(const Size &input)\n> +{\n> +\tif (input.width < 2 || input.height < 2) {\n> +\t\tLOG(Converter, Error)\n> +\t\t\t<< \"Input format size too small: \" << input.toString();\n> +\t\treturn {};\n> +\t}\n> +\n> +\treturn SizeRange(Size(input.width - 2, input.height - 2));\n> +}\n> +\n> +std::tuple\n> +SwConverter::strideAndFrameSize(const PixelFormat &pixelFormat,\n> +\t\t\t\tconst Size &size)\n> +{\n> +\t/* Only RGB888 output is currently supported */\n> +\tif (pixelFormat == formats::RGB888) {\n> +\t\tint stride = size.width * 3;\n> +\t\treturn std::make_tuple(stride, stride * (size.height));\n> +\t}\n> +\n> +\treturn std::make_tuple(0, 0);\n\nWhat about:\n\n\t/* Only RGB888 output is currently supported */\n\tif (pixelFormat != formats::RGB888)\n\t\treturn {};\n\n\tunsigned int stride = size.width * 3;\n\treturn std::make_tuple(stride, stride * size.height);\n\nUp to you, really.\n\n> +}\n> +\n> +int SwConverter::configure(const StreamConfiguration &inputCfg,\n> +\t\t\t const std::vector> &outputCfgs)\n> +{\n> +\tif (outputCfgs.size() != 1) {\n> +\t\tLOG(Converter, Error)\n> +\t\t\t<< \"Unsupported number of output streams: \"\n> +\t\t\t<< outputCfgs.size();\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tisp_ = std::make_unique(this);\n\nWhy do you re-create the isp_ at every configuration ?\n\nI think the actual soft ISP implementation could be discussed f2f and\nI'll stop here with the nitpicking ;)\n\nThanks\n j\n\n> +\n> +\treturn isp_->configure(inputCfg, outputCfgs[0]);\n> +}\n> +\n> +int SwConverter::Isp::configure(const StreamConfiguration &inputCfg,\n> +\t\tconst StreamConfiguration &outputCfg)\n> +{\n> +\tBayerFormat bayerFormat =\n> +\t\tBayerFormat::fromPixelFormat(inputCfg.pixelFormat);\n> +\twidth_ = inputCfg.size.width;\n> +\theight_ = inputCfg.size.height;\n> +\tstride_ = inputCfg.stride;\n> +\n> +\tif (bayerFormat.bitDepth != 10 ||\n> +\t bayerFormat.packing != BayerFormat::Packing::CSI2 ||\n> +\t width_ < 2 || height_ < 2) {\n> +\t\tLOG(Converter, Error) << \"Input format \"\n> +\t\t\t\t << inputCfg.size << \"-\"\n> +\t\t\t\t << inputCfg.pixelFormat\n> +\t\t\t\t << \"not supported\";\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tswitch (bayerFormat.order) {\n> +\tcase BayerFormat::BGGR:\n> +\t\tred_shift_ = Point(0, 0);\n> +\t\tbreak;\n> +\tcase BayerFormat::GBRG:\n> +\t\tred_shift_ = Point(1, 0);\n> +\t\tbreak;\n> +\tcase BayerFormat::GRBG:\n> +\t\tred_shift_ = Point(0, 1);\n> +\t\tbreak;\n> +\tcase BayerFormat::RGGB:\n> +\tdefault:\n> +\t\tred_shift_ = Point(1, 1);\n> +\t\tbreak;\n> +\t}\n> +\n> +\tif (outputCfg.size.width != width_ - 2 ||\n> +\t outputCfg.size.height != height_ - 2 ||\n> +\t outputCfg.stride != (width_ - 2) * 3 ||\n> +\t outputCfg.pixelFormat != formats::RGB888) {\n> +\t\tLOG(Converter, Error)\n> +\t\t\t<< \"Output format not supported\";\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tLOG(Converter, Info) << \"SwConverter configuration: \"\n> +\t\t\t << inputCfg.size << \"-\" << inputCfg.pixelFormat\n> +\t\t\t << \" -> \"\n> +\t\t\t << outputCfg.size << \"-\" << outputCfg.pixelFormat;\n> +\n> +\t/* set r/g/b gains to 1.0 until frame data collected */\n> +\trNumerat_ = rDenomin_ = 1;\n> +\tbNumerat_ = bDenomin_ = 1;\n> +\tgNumerat_ = gDenomin_ = 1;\n> +\n> +\treturn 0;\n> +}\n> +\n> +int SwConverter::exportBuffers(unsigned int output, unsigned int count,\n> +\t\t\t std::vector> *buffers)\n> +{\n> +\t/* single output for now */\n> +\tif (output >= 1)\n> +\t\treturn -EINVAL;\n> +\n> +\treturn isp_->exportBuffers(count, buffers);\n> +}\n> +\n> +int SwConverter::Isp::exportBuffers(unsigned int count,\n> +\t\t\t\t std::vector> *buffers)\n> +{\n> +\t/* V4L2_PIX_FMT_BGR24 aka 'BGR3' for output: */\n> +\tunsigned int bufSize = (height_ - 2) * (width_ - 2) * 3;\n> +\n> +\tfor (unsigned int i = 0; i < count; i++) {\n> +\t\tstd::string name = \"frame-\" + std::to_string(i);\n> +\n> +\t\tconst int ispFd = memfd_create(name.c_str(), 0);\n> +\t\tint ret = ftruncate(ispFd, bufSize);\n> +\t\tif (ret < 0) {\n> +\t\t\tLOG(Converter, Error) << \"ftruncate() for memfd failed \"\n> +\t\t\t\t\t << strerror(-ret);\n> +\t\t\treturn ret;\n> +\t\t}\n> +\n> +\t\tFrameBuffer::Plane outPlane;\n> +\t\toutPlane.fd = SharedFD(std::move(ispFd));\n> +\t\toutPlane.offset = 0;\n> +\t\toutPlane.length = bufSize;\n> +\n> +\t\tstd::vector planes{ outPlane };\n> +\t\tbuffers->emplace_back(std::make_unique(std::move(planes)));\n> +\t}\n> +\n> +\treturn count;\n> +}\n> +\n> +int SwConverter::Isp::start()\n> +{\n> +\tmoveToThread(&thread_);\n> +\tthread_.start();\n> +\treturn 0;\n> +}\n> +\n> +void SwConverter::Isp::stop()\n> +{\n> +\tthread_.exit();\n> +\tthread_.wait();\n> +}\n> +\n> +int SwConverter::start()\n> +{\n> +\treturn isp_->start();\n> +}\n> +\n> +void SwConverter::stop()\n> +{\n> +\treturn isp_->stop();\n> +}\n> +\n> +int SwConverter::queueBuffers(FrameBuffer *input,\n> +\t\t\t const std::map &outputs)\n> +{\n> +\tunsigned int mask = 0;\n> +\n> +\t/*\n> +\t * Validate the outputs as a sanity check: at least one output is\n> +\t * required, all outputs must reference a valid stream and no two\n> +\t * outputs can reference the same stream.\n> +\t */\n> +\tif (outputs.empty())\n> +\t\treturn -EINVAL;\n> +\n> +\tfor (auto [index, buffer] : outputs) {\n> +\t\tif (!buffer)\n> +\t\t\treturn -EINVAL;\n> +\t\tif (index >= 1) /* only single stream atm */\n> +\t\t\treturn -EINVAL;\n> +\t\tif (mask & (1 << index))\n> +\t\t\treturn -EINVAL;\n> +\n> +\t\tmask |= 1 << index;\n> +\t}\n> +\n> +\tprocess(input, outputs.at(0));\n> +\n> +\treturn 0;\n> +}\n> +\n> +void SwConverter::process(FrameBuffer *input, FrameBuffer *output)\n> +{\n> +\tisp_->invokeMethod(&SwConverter::Isp::process,\n> +\t\t\t ConnectionTypeQueued, input, output);\n> +}\n> +\n> +void SwConverter::Isp::process(FrameBuffer *input, FrameBuffer *output)\n> +{\n> +\t/* Copy metadata from the input buffer */\n> +\tFrameMetadata &metadata = output->_d()->metadata();\n> +\tmetadata.status = input->metadata().status;\n> +\tmetadata.sequence = input->metadata().sequence;\n> +\tmetadata.timestamp = input->metadata().timestamp;\n> +\n> +\tMappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read);\n> +\tMappedFrameBuffer out(output, MappedFrameBuffer::MapFlag::Write);\n> +\tif (!in.isValid() || !out.isValid()) {\n> +\t\tLOG(Converter, Error) << \"mmap-ing buffer(s) failed\";\n> +\t\tmetadata.status = FrameMetadata::FrameError;\n> +\t\tconverter_->outputBufferReady.emit(output);\n> +\t\tconverter_->inputBufferReady.emit(input);\n> +\t\treturn;\n> +\t}\n> +\n> +\tdebayer(out.planes()[0].data(), in.planes()[0].data());\n> +\tmetadata.planes()[0].bytesused = out.planes()[0].size();\n> +\n> +\tconverter_->outputBufferReady.emit(output);\n> +\tconverter_->inputBufferReady.emit(input);\n> +}\n> +\n> +void SwConverter::Isp::debayer(uint8_t *dst, const uint8_t *src)\n> +{\n> +\t/* RAW10P input format is assumed */\n> +\n> +\t/* output buffer is in BGR24 format and is of (width-2)*(height-2) */\n> +\n> +\tint w_out = width_ - 2;\n> +\tint h_out = height_ - 2;\n> +\n> +\tunsigned long sumR = 0;\n> +\tunsigned long sumB = 0;\n> +\tunsigned long sumG = 0;\n> +\n> +\tfor (int y = 0; y < h_out; y++) {\n> +\t\tconst uint8_t *pin_base = src + (y + 1) * stride_;\n> +\t\tuint8_t *pout = dst + y * w_out * 3;\n> +\t\tint phase_y = (y + red_shift_.y) % 2;\n> +\n> +\t\tfor (int x = 0; x < w_out; x++) {\n> +\t\t\tint phase_x = (x + red_shift_.x) % 2;\n> +\t\t\tint phase = 2 * phase_y + phase_x;\n> +\n> +\t\t\t/* x part of the offset in the input buffer: */\n> +\t\t\tint x_m1 = x + x / 4;\t\t/* offset for (x-1) */\n> +\t\t\tint x_0 = x + 1 + (x + 1) / 4;\t/* offset for x */\n> +\t\t\tint x_p1 = x + 2 + (x + 2) / 4;\t/* offset for (x+1) */\n> +\t\t\t/* the colour component value to write to the output */\n> +\t\t\tunsigned val;\n> +\n> +\t\t\tswitch (phase) {\n> +\t\t\tcase 0: /* at R pixel */\n> +\t\t\t\t/* blue: ((-1,-1)+(1,-1)+(-1,1)+(1,1)) / 4 */\n> +\t\t\t\tval = ( *(pin_base + x_m1 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_m1 + stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1 + stride_) ) >> 2;\n> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* green: ((0,-1)+(-1,0)+(1,0)+(0,1)) / 4 */\n> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1)\n> +\t\t\t\t\t+ *(pin_base + x_m1)\n> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 2;\n> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* red: (0,0) */\n> +\t\t\t\tval = *(pin_base + x_0);\n> +\t\t\t\tsumR += val;\n> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\tbreak;\n> +\t\t\tcase 1: /* at Gr pixel */\n> +\t\t\t\t/* blue: ((0,-1) + (0,1)) / 2 */\n> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 1;\n> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* green: (0,0) */\n> +\t\t\t\tval = *(pin_base + x_0);\n> +\t\t\t\tsumG += val;\n> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* red: ((-1,0) + (1,0)) / 2 */\n> +\t\t\t\tval = ( *(pin_base + x_m1)\n> +\t\t\t\t\t+ *(pin_base + x_p1) ) >> 1;\n> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\tbreak;\n> +\t\t\tcase 2: /* at Gb pixel */\n> +\t\t\t\t/* blue: ((-1,0) + (1,0)) / 2 */\n> +\t\t\t\tval = ( *(pin_base + x_m1)\n> +\t\t\t\t\t+ *(pin_base + x_p1) ) >> 1;\n> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* green: (0,0) */\n> +\t\t\t\tval = *(pin_base + x_0);\n> +\t\t\t\tsumG += val;\n> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* red: ((0,-1) + (0,1)) / 2 */\n> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 1;\n> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\tbreak;\n> +\t\t\tdefault: /* at B pixel */\n> +\t\t\t\t/* blue: (0,0) */\n> +\t\t\t\tval = *(pin_base + x_0);\n> +\t\t\t\tsumB += val;\n> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* green: ((0,-1)+(-1,0)+(1,0)+(0,1)) / 4 */\n> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1)\n> +\t\t\t\t\t+ *(pin_base + x_m1)\n> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 2;\n> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t\t/* red: ((-1,-1)+(1,-1)+(-1,1)+(1,1)) / 4 */\n> +\t\t\t\tval = ( *(pin_base + x_m1 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1 - stride_)\n> +\t\t\t\t\t+ *(pin_base + x_m1 + stride_)\n> +\t\t\t\t\t+ *(pin_base + x_p1 + stride_) ) >> 2;\n> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n> +\t\t\t}\n> +\t\t}\n> +\t}\n> +\n> +\t/* calculate red and blue gains for simple AWB */\n> +\tLOG(Converter, Debug) << \"sumR = \" << sumR\n> +\t\t\t << \", sumB = \" << sumB << \", sumG = \" << sumG;\n> +\n> +\tsumG /= 2; /* the number of G pixels is twice as big vs R and B ones */\n> +\n> +\t/* normalize red, blue, and green sums to fit into 22-bit value */\n> +\tunsigned long fRed = sumR / 0x400000;\n> +\tunsigned long fBlue = sumB / 0x400000;\n> +\tunsigned long fGreen = sumG / 0x400000;\n> +\tunsigned long fNorm = std::max({ 1UL, fRed, fBlue, fGreen });\n> +\tsumR /= fNorm;\n> +\tsumB /= fNorm;\n> +\tsumG /= fNorm;\n> +\n> +\tLOG(Converter, Debug) << \"fNorm = \" << fNorm;\n> +\tLOG(Converter, Debug) << \"Normalized: sumR = \" << sumR\n> +\t\t\t << \", sumB= \" << sumB << \", sumG = \" << sumG;\n> +\n> +\t/* make sure red/blue gains never exceed approximately 256 */\n> +\tunsigned long minDenom;\n> +\trNumerat_ = (sumR + sumB + sumG) / 3;\n> +\tminDenom = rNumerat_ / 0x100;\n> +\trDenomin_ = std::max(minDenom, sumR);\n> +\tbNumerat_ = rNumerat_;\n> +\tbDenomin_ = std::max(minDenom, sumB);\n> +\tgNumerat_ = rNumerat_;\n> +\tgDenomin_ = std::max(minDenom, sumG);\n> +\n> +\tLOG(Converter, Debug) << \"rGain = [ \"\n> +\t\t\t << rNumerat_ << \" / \" << rDenomin_\n> +\t\t\t << \" ], bGain = [ \" << bNumerat_ << \" / \" << bDenomin_\n> +\t\t\t << \" ], gGain = [ \" << gNumerat_ << \" / \" << gDenomin_\n> +\t\t\t << \" (minDenom = \" << minDenom << \")\";\n> +}\n> +\n> +static std::initializer_list compatibles = {};\n> +\n> +REGISTER_CONVERTER(\"linaro-sw-converter\", SwConverter, compatibles)\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/converter/meson.build b/src/libcamera/converter/meson.build\n> index 2aa72fe4..1f1e0ea4 100644\n> --- a/src/libcamera/converter/meson.build\n> +++ b/src/libcamera/converter/meson.build\n> @@ -1,5 +1,6 @@\n> # SPDX-License-Identifier: CC0-1.0\n>\n> libcamera_sources += files([\n> - 'converter_v4l2_m2m.cpp'\n> + 'converter_softw.cpp',\n> + 'converter_v4l2_m2m.cpp',\n> ])\n> --\n> 2.34.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id EED4BC326B\n\tfor ;\n\tThu, 21 Sep 2023 10:40:36 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 2100462944;\n\tThu, 21 Sep 2023 12:40:36 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id F2F2F628D8\n\tfor ;\n\tThu, 21 Sep 2023 12:40:33 +0200 (CEST)","from ideasonboard.com (93-46-82-201.ip106.fastwebnet.it\n\t[93.46.82.201])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 7BFA7B75;\n\tThu, 21 Sep 2023 12:38:56 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1695292836;\n\tbh=u8JX4BkdAopOYCK4JP24uBsDd/bIQAqetmA18FgciUQ=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=1t0v+vbmjyyNw94Ene0eMWh6IYtw8VkLjoKlPObNZd1gP/fGT1S1PrSuKCei8Opua\n\tJ9OLZyqEiKqmmjj8x3ppadp+2zIXdn00b3lnThxobUCzF//4dqaZqHcEjUh1IT1dp/\n\tpJjV9qYg7YwlikjjsBjyoybPRCiHnlN6JrsEpDOIfuHstUKKfj1OrII99DHh2B0Uvf\n\tjfJURqi60Q5c5PDWkJcwK0nHV73GymqNCiyT1m/xyNKRAu+iJCGero71sQgu9tlDJZ\n\tYYgteuiHR1egrMXTSdwPT7vT/XoqJuOqfLoeMOKw/Cqeo8fpeg/e66LLLqxDK/D7n/\n\tk0sZffFG1CRxw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1695292736;\n\tbh=u8JX4BkdAopOYCK4JP24uBsDd/bIQAqetmA18FgciUQ=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=BvcWrWF3AR+UuM6TBmyyhFi9cRqsc8bmPPjLXhim+2VGDgSXZ5Fs5zGE5BVbe7QDr\n\tM6DghY1XzAMWeKV/HV00Pc7QvFNFe7xWbijfeYbwG7XOrURW+jB+mbINQkyVyMhHQe\n\tGfrBwrJT9ugPFAALLWD8Ur4VjPLguKOmkESCG3o0="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"BvcWrWF3\"; dkim-atps=neutral","Date":"Thu, 21 Sep 2023 12:40:31 +0200","To":"Andrey Konovalov ","Message-ID":"","References":"<20230920151921.31273-1-andrey.konovalov@linaro.org>\n\t<20230920151921.31273-4-andrey.konovalov@linaro.org>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20230920151921.31273-4-andrey.konovalov@linaro.org>","Subject":"Re: [libcamera-devel] [RFC PATCH v2 3/4] libcamera: converter: add\n\tsoftware converter","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Jacopo Mondi via libcamera-devel ","Reply-To":"Jacopo Mondi ","Cc":"jacopo.mondi@ideasonboard.com, bryan.odonoghue@linaro.org,\n\tlibcamera-devel@lists.libcamera.org, srinivas.kandagatla@linaro.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":27840,"web_url":"https://patchwork.libcamera.org/comment/27840/","msgid":"","date":"2023-09-21T19:13:23","subject":"Re: [libcamera-devel] [RFC PATCH v2 3/4] libcamera: converter: add\n\tsoftware converter","submitter":{"id":25,"url":"https://patchwork.libcamera.org/api/people/25/","name":"Andrey Konovalov","email":"andrey.konovalov@linaro.org"},"content":"Hi Jacopo,\n\nThank you for the review!\n\nOn 21.09.2023 13:40, Jacopo Mondi wrote:\n> Hi Andrey\n> \n> Sw converters will be discussed in detail, so I'll only go through the\n> trivial/cosmetics things and leave discussion on the architecture for\n> the face-2-face meeting in Paris\n\nSounds good!\n\n> On Wed, Sep 20, 2023 at 06:19:20PM +0300, Andrey Konovalov via libcamera-devel wrote:\n>> Use the Converter interface to implement bilinear Bayer demosaic\n>> filtering and very simple AWB on CPU.\n>>\n>> The only output format currently supported is RGB888.\n>>\n>> Signed-off-by: Andrey Konovalov \n>> ---\n>> .../internal/converter/converter_softw.h | 90 ++++\n>> .../libcamera/internal/converter/meson.build | 1 +\n>> src/libcamera/converter/converter_softw.cpp | 410 ++++++++++++++++++\n>> src/libcamera/converter/meson.build | 3 +-\n>> 4 files changed, 503 insertions(+), 1 deletion(-)\n>> create mode 100644 include/libcamera/internal/converter/converter_softw.h\n>> create mode 100644 src/libcamera/converter/converter_softw.cpp\n>>\n>> diff --git a/include/libcamera/internal/converter/converter_softw.h b/include/libcamera/internal/converter/converter_softw.h\n>> new file mode 100644\n>> index 00000000..eff9d6b2\n>> --- /dev/null\n>> +++ b/include/libcamera/internal/converter/converter_softw.h\n>> @@ -0,0 +1,90 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2023, Linaro Ltd\n>> + *\n>> + * converter_softw.h - interface of software converter (runs 100% on CPU)\n>> + */\n>> +\n> \n> #include \n> \n>> +#include \n> \n> #include \n> #include \n\nOK\n\n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +\n>> +#include \"libcamera/internal/converter.h\"\n>> +\n>> +namespace libcamera {\n>> +\n>> +class FrameBuffer;\n>> +class MediaDevice;\n>> +class Size;\n>> +class SizeRange;\n>> +struct StreamConfiguration;\n>> +\n>> +class SwConverter : public Converter\n>> +{\n>> +public:\n>> +\tSwConverter([[maybe_unused]] MediaDevice *media)\n> \n> This is not super-nice,\n\nAgreed\n\n> but I don't have anything better to suggest at\n> thsi time unless we completely revisit the implementation of [2/4]. I\n> guess we can live with this even if it might indicate we should be\n> doing better.\n> \n> I had a go at using SFINAE to selectively compile in the version of\n> ConverterFactory::createInstance() that passes in a MediaDevice\n> selectively, but after 30 minutes I gave up :)\n\nI can't say I am very familiar with SFINAE, but will see if I can get to\nsomething that works...\n\n>> +\t\t: Converter() {}\n>> +\n>> +\tint loadConfiguration([[maybe_unused]] const std::string &filename) { return 0; }\n>> +\tbool isValid() const { return true; }\n>> +\n>> +\tstd::vector formats(PixelFormat input);\n>> +\tSizeRange sizes(const Size &input);\n>> +\n>> +\tstd::tuple\n>> +\tstrideAndFrameSize(const PixelFormat &pixelFormat, const Size &size);\n>> +\n>> +\tint configure(const StreamConfiguration &inputCfg,\n>> +\t\t const std::vector> &outputCfg);\n>> +\tint exportBuffers(unsigned int ouput, unsigned int count,\n>> +\t\t\t std::vector> *buffers);\n>> +\n>> +\tvoid process(FrameBuffer *input, FrameBuffer *output);\n>> +\tint start();\n>> +\tvoid stop();\n>> +\n>> +\tint queueBuffers(FrameBuffer *input,\n>> +\t\t\t const std::map &outputs);\n>> +\n>> +private:\n>> +\tclass Isp : public Object\n>> +\t{\n>> +\tpublic:\n>> +\t\tIsp(SwConverter *converter)\n>> +\t\t\t: converter_(converter) {}\n>> +\n>> +\t\tint configure(const StreamConfiguration &inputCfg,\n>> +\t\t\t const StreamConfiguration &outputCfg);\n>> +\t\tint exportBuffers(unsigned int count,\n>> +\t\t\t\t std::vector> *buffers);\n>> +\t\tvoid process(FrameBuffer *input, FrameBuffer *output);\n>> +\t\tint start();\n>> +\t\tvoid stop();\n>> +\n>> +\tprivate:\n>> +\t\tvoid debayer(uint8_t *dst, const uint8_t *src);\n>> +\n>> +\t\tSwConverter *converter_;\n>> +\n>> +\t\tThread thread_;\n>> +\n>> +\t\tunsigned int width_;\n>> +\t\tunsigned int height_;\n>> +\t\tunsigned int stride_;\n>> +\t\tPoint red_shift_;\n>> +\n>> +\t\tunsigned long rNumerat_, rDenomin_; /* red gain for AWB */\n>> +\t\tunsigned long bNumerat_, bDenomin_; /* blue gain for AWB */\n>> +\t\tunsigned long gNumerat_, gDenomin_; /* green gain for AWB */\n>> +\t};\n>> +\n>> +\tstd::unique_ptr isp_;\n>> +};\n>> +\n>> +} /* namespace libcamera */\n>> diff --git a/include/libcamera/internal/converter/meson.build b/include/libcamera/internal/converter/meson.build\n>> index 891e79e7..843a0483 100644\n>> --- a/include/libcamera/internal/converter/meson.build\n>> +++ b/include/libcamera/internal/converter/meson.build\n>> @@ -1,5 +1,6 @@\n>> # SPDX-License-Identifier: CC0-1.0\n>>\n>> libcamera_internal_headers += files([\n>> + 'converter_softw.h',\n>> 'converter_v4l2_m2m.h',\n>> ])\n>> diff --git a/src/libcamera/converter/converter_softw.cpp b/src/libcamera/converter/converter_softw.cpp\n>> new file mode 100644\n>> index 00000000..ad7e6f3d\n>> --- /dev/null\n>> +++ b/src/libcamera/converter/converter_softw.cpp\n>> @@ -0,0 +1,410 @@\n>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n>> +/*\n>> + * Copyright (C) 2023, Linaro Ltd\n>> + *\n>> + * converter_softw.h - interface of software converter (runs 100% on CPU)\n>> + */\n>> +\n>> +#include \n>> +#include \n>> +#include \n>> +#include \n>> +#include \n>> +\n>> +#include \n>> +#include \n>> +\n>> +#include \"libcamera/internal/bayer_format.h\"\n>> +#include \"libcamera/internal/framebuffer.h\"\n>> +#include \"libcamera/internal/mapped_framebuffer.h\"\n>> +\n>> +#include \"libcamera/internal/converter/converter_softw.h\"\n> \n> Include this first to make sure it's self contained\n> \n> And you can remove a few duplicated includes like \n\nOK\n\n>> +\n>> +namespace libcamera {\n>> +\n>> +LOG_DECLARE_CATEGORY(Converter)\n>> +\n>> +std::vector SwConverter::formats(PixelFormat input)\n>> +{\n>> +\tstd::vector pixelFormats;\n>> +\tBayerFormat inputFormat = BayerFormat::fromPixelFormat(input);\n>> +\n>> +\t/* Only RAW10P is currently supported */\n> \n> Do you care about the bayer ordering or any permutation is fine\n> (SRGGB10, SBGGR10 etc etc)\n\nAll the four permutations are supported.\n\n>> +\tif (inputFormat.bitDepth == 10 && inputFormat.packing == BayerFormat::Packing::CSI2)\n>> +\t\tpixelFormats.push_back(formats::RGB888);\n>> +\n>> +\tif (pixelFormats.empty())\n>> +\t\tLOG(Converter, Info)\n>> +\t\t\t<< \"Unsupported input format \" << input.toString();\n>> +\n>> +\treturn pixelFormats;\n> \n> Isn't\n> \tBayerFormat inputFormat = BayerFormat::fromPixelFormat(input);\n> \tif (inputFormat.bitDepth != 10 ||\n> \t inputFormat.packing != BayerFormat::Packing::CSI2) {\n> \t\tLOG(Converter, Info)\n> \t\t\t<< \"Unsupported input format \" << input.toString();\n> return {};\n> \t}\n> \n> return { formats::RGB888 };\n> \n> a bit simpler ?\n\nYes, it is.\n\nWe had plans to support more than single output format, but this is low priority\nat the moment. So 'return { formats::RGB888 };' would be fine for now.\n\n>> +}\n>> +\n>> +SizeRange SwConverter::sizes(const Size &input)\n>> +{\n>> +\tif (input.width < 2 || input.height < 2) {\n>> +\t\tLOG(Converter, Error)\n>> +\t\t\t<< \"Input format size too small: \" << input.toString();\n>> +\t\treturn {};\n>> +\t}\n>> +\n>> +\treturn SizeRange(Size(input.width - 2, input.height - 2));\n>> +}\n>> +\n>> +std::tuple\n>> +SwConverter::strideAndFrameSize(const PixelFormat &pixelFormat,\n>> +\t\t\t\tconst Size &size)\n>> +{\n>> +\t/* Only RGB888 output is currently supported */\n>> +\tif (pixelFormat == formats::RGB888) {\n>> +\t\tint stride = size.width * 3;\n>> +\t\treturn std::make_tuple(stride, stride * (size.height));\n>> +\t}\n>> +\n>> +\treturn std::make_tuple(0, 0);\n> \n> What about:\n> \n> \t/* Only RGB888 output is currently supported */\n> \tif (pixelFormat != formats::RGB888)\n> \t\treturn {};\n> \n> \tunsigned int stride = size.width * 3;\n> \treturn std::make_tuple(stride, stride * size.height);\n> \n> Up to you, really.\n\nOK, I can change that.\n\n>> +}\n>> +\n>> +int SwConverter::configure(const StreamConfiguration &inputCfg,\n>> +\t\t\t const std::vector> &outputCfgs)\n>> +{\n>> +\tif (outputCfgs.size() != 1) {\n>> +\t\tLOG(Converter, Error)\n>> +\t\t\t<< \"Unsupported number of output streams: \"\n>> +\t\t\t<< outputCfgs.size();\n>> +\t\treturn -EINVAL;\n>> +\t}\n>> +\n>> +\tisp_ = std::make_unique(this);\n> \n> Why do you re-create the isp_ at every configuration ?\n\nOops...\nLet's pretend I didn't put it here, and you haven't seen that line :)\n(Will fix that of course; thanks for noticing)\n\n> I think the actual soft ISP implementation could be discussed f2f and\n> I'll stop here with the nitpicking ;)\n\nThat's fine. Thanks for reviewing this stuff!\n\nThanks,\nAndrey\n\n> Thanks\n> j\n> \n>> +\n>> +\treturn isp_->configure(inputCfg, outputCfgs[0]);\n>> +}\n>> +\n>> +int SwConverter::Isp::configure(const StreamConfiguration &inputCfg,\n>> +\t\tconst StreamConfiguration &outputCfg)\n>> +{\n>> +\tBayerFormat bayerFormat =\n>> +\t\tBayerFormat::fromPixelFormat(inputCfg.pixelFormat);\n>> +\twidth_ = inputCfg.size.width;\n>> +\theight_ = inputCfg.size.height;\n>> +\tstride_ = inputCfg.stride;\n>> +\n>> +\tif (bayerFormat.bitDepth != 10 ||\n>> +\t bayerFormat.packing != BayerFormat::Packing::CSI2 ||\n>> +\t width_ < 2 || height_ < 2) {\n>> +\t\tLOG(Converter, Error) << \"Input format \"\n>> +\t\t\t\t << inputCfg.size << \"-\"\n>> +\t\t\t\t << inputCfg.pixelFormat\n>> +\t\t\t\t << \"not supported\";\n>> +\t\treturn -EINVAL;\n>> +\t}\n>> +\n>> +\tswitch (bayerFormat.order) {\n>> +\tcase BayerFormat::BGGR:\n>> +\t\tred_shift_ = Point(0, 0);\n>> +\t\tbreak;\n>> +\tcase BayerFormat::GBRG:\n>> +\t\tred_shift_ = Point(1, 0);\n>> +\t\tbreak;\n>> +\tcase BayerFormat::GRBG:\n>> +\t\tred_shift_ = Point(0, 1);\n>> +\t\tbreak;\n>> +\tcase BayerFormat::RGGB:\n>> +\tdefault:\n>> +\t\tred_shift_ = Point(1, 1);\n>> +\t\tbreak;\n>> +\t}\n>> +\n>> +\tif (outputCfg.size.width != width_ - 2 ||\n>> +\t outputCfg.size.height != height_ - 2 ||\n>> +\t outputCfg.stride != (width_ - 2) * 3 ||\n>> +\t outputCfg.pixelFormat != formats::RGB888) {\n>> +\t\tLOG(Converter, Error)\n>> +\t\t\t<< \"Output format not supported\";\n>> +\t\treturn -EINVAL;\n>> +\t}\n>> +\n>> +\tLOG(Converter, Info) << \"SwConverter configuration: \"\n>> +\t\t\t << inputCfg.size << \"-\" << inputCfg.pixelFormat\n>> +\t\t\t << \" -> \"\n>> +\t\t\t << outputCfg.size << \"-\" << outputCfg.pixelFormat;\n>> +\n>> +\t/* set r/g/b gains to 1.0 until frame data collected */\n>> +\trNumerat_ = rDenomin_ = 1;\n>> +\tbNumerat_ = bDenomin_ = 1;\n>> +\tgNumerat_ = gDenomin_ = 1;\n>> +\n>> +\treturn 0;\n>> +}\n>> +\n>> +int SwConverter::exportBuffers(unsigned int output, unsigned int count,\n>> +\t\t\t std::vector> *buffers)\n>> +{\n>> +\t/* single output for now */\n>> +\tif (output >= 1)\n>> +\t\treturn -EINVAL;\n>> +\n>> +\treturn isp_->exportBuffers(count, buffers);\n>> +}\n>> +\n>> +int SwConverter::Isp::exportBuffers(unsigned int count,\n>> +\t\t\t\t std::vector> *buffers)\n>> +{\n>> +\t/* V4L2_PIX_FMT_BGR24 aka 'BGR3' for output: */\n>> +\tunsigned int bufSize = (height_ - 2) * (width_ - 2) * 3;\n>> +\n>> +\tfor (unsigned int i = 0; i < count; i++) {\n>> +\t\tstd::string name = \"frame-\" + std::to_string(i);\n>> +\n>> +\t\tconst int ispFd = memfd_create(name.c_str(), 0);\n>> +\t\tint ret = ftruncate(ispFd, bufSize);\n>> +\t\tif (ret < 0) {\n>> +\t\t\tLOG(Converter, Error) << \"ftruncate() for memfd failed \"\n>> +\t\t\t\t\t << strerror(-ret);\n>> +\t\t\treturn ret;\n>> +\t\t}\n>> +\n>> +\t\tFrameBuffer::Plane outPlane;\n>> +\t\toutPlane.fd = SharedFD(std::move(ispFd));\n>> +\t\toutPlane.offset = 0;\n>> +\t\toutPlane.length = bufSize;\n>> +\n>> +\t\tstd::vector planes{ outPlane };\n>> +\t\tbuffers->emplace_back(std::make_unique(std::move(planes)));\n>> +\t}\n>> +\n>> +\treturn count;\n>> +}\n>> +\n>> +int SwConverter::Isp::start()\n>> +{\n>> +\tmoveToThread(&thread_);\n>> +\tthread_.start();\n>> +\treturn 0;\n>> +}\n>> +\n>> +void SwConverter::Isp::stop()\n>> +{\n>> +\tthread_.exit();\n>> +\tthread_.wait();\n>> +}\n>> +\n>> +int SwConverter::start()\n>> +{\n>> +\treturn isp_->start();\n>> +}\n>> +\n>> +void SwConverter::stop()\n>> +{\n>> +\treturn isp_->stop();\n>> +}\n>> +\n>> +int SwConverter::queueBuffers(FrameBuffer *input,\n>> +\t\t\t const std::map &outputs)\n>> +{\n>> +\tunsigned int mask = 0;\n>> +\n>> +\t/*\n>> +\t * Validate the outputs as a sanity check: at least one output is\n>> +\t * required, all outputs must reference a valid stream and no two\n>> +\t * outputs can reference the same stream.\n>> +\t */\n>> +\tif (outputs.empty())\n>> +\t\treturn -EINVAL;\n>> +\n>> +\tfor (auto [index, buffer] : outputs) {\n>> +\t\tif (!buffer)\n>> +\t\t\treturn -EINVAL;\n>> +\t\tif (index >= 1) /* only single stream atm */\n>> +\t\t\treturn -EINVAL;\n>> +\t\tif (mask & (1 << index))\n>> +\t\t\treturn -EINVAL;\n>> +\n>> +\t\tmask |= 1 << index;\n>> +\t}\n>> +\n>> +\tprocess(input, outputs.at(0));\n>> +\n>> +\treturn 0;\n>> +}\n>> +\n>> +void SwConverter::process(FrameBuffer *input, FrameBuffer *output)\n>> +{\n>> +\tisp_->invokeMethod(&SwConverter::Isp::process,\n>> +\t\t\t ConnectionTypeQueued, input, output);\n>> +}\n>> +\n>> +void SwConverter::Isp::process(FrameBuffer *input, FrameBuffer *output)\n>> +{\n>> +\t/* Copy metadata from the input buffer */\n>> +\tFrameMetadata &metadata = output->_d()->metadata();\n>> +\tmetadata.status = input->metadata().status;\n>> +\tmetadata.sequence = input->metadata().sequence;\n>> +\tmetadata.timestamp = input->metadata().timestamp;\n>> +\n>> +\tMappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read);\n>> +\tMappedFrameBuffer out(output, MappedFrameBuffer::MapFlag::Write);\n>> +\tif (!in.isValid() || !out.isValid()) {\n>> +\t\tLOG(Converter, Error) << \"mmap-ing buffer(s) failed\";\n>> +\t\tmetadata.status = FrameMetadata::FrameError;\n>> +\t\tconverter_->outputBufferReady.emit(output);\n>> +\t\tconverter_->inputBufferReady.emit(input);\n>> +\t\treturn;\n>> +\t}\n>> +\n>> +\tdebayer(out.planes()[0].data(), in.planes()[0].data());\n>> +\tmetadata.planes()[0].bytesused = out.planes()[0].size();\n>> +\n>> +\tconverter_->outputBufferReady.emit(output);\n>> +\tconverter_->inputBufferReady.emit(input);\n>> +}\n>> +\n>> +void SwConverter::Isp::debayer(uint8_t *dst, const uint8_t *src)\n>> +{\n>> +\t/* RAW10P input format is assumed */\n>> +\n>> +\t/* output buffer is in BGR24 format and is of (width-2)*(height-2) */\n>> +\n>> +\tint w_out = width_ - 2;\n>> +\tint h_out = height_ - 2;\n>> +\n>> +\tunsigned long sumR = 0;\n>> +\tunsigned long sumB = 0;\n>> +\tunsigned long sumG = 0;\n>> +\n>> +\tfor (int y = 0; y < h_out; y++) {\n>> +\t\tconst uint8_t *pin_base = src + (y + 1) * stride_;\n>> +\t\tuint8_t *pout = dst + y * w_out * 3;\n>> +\t\tint phase_y = (y + red_shift_.y) % 2;\n>> +\n>> +\t\tfor (int x = 0; x < w_out; x++) {\n>> +\t\t\tint phase_x = (x + red_shift_.x) % 2;\n>> +\t\t\tint phase = 2 * phase_y + phase_x;\n>> +\n>> +\t\t\t/* x part of the offset in the input buffer: */\n>> +\t\t\tint x_m1 = x + x / 4;\t\t/* offset for (x-1) */\n>> +\t\t\tint x_0 = x + 1 + (x + 1) / 4;\t/* offset for x */\n>> +\t\t\tint x_p1 = x + 2 + (x + 2) / 4;\t/* offset for (x+1) */\n>> +\t\t\t/* the colour component value to write to the output */\n>> +\t\t\tunsigned val;\n>> +\n>> +\t\t\tswitch (phase) {\n>> +\t\t\tcase 0: /* at R pixel */\n>> +\t\t\t\t/* blue: ((-1,-1)+(1,-1)+(-1,1)+(1,1)) / 4 */\n>> +\t\t\t\tval = ( *(pin_base + x_m1 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_m1 + stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1 + stride_) ) >> 2;\n>> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* green: ((0,-1)+(-1,0)+(1,0)+(0,1)) / 4 */\n>> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1)\n>> +\t\t\t\t\t+ *(pin_base + x_m1)\n>> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 2;\n>> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* red: (0,0) */\n>> +\t\t\t\tval = *(pin_base + x_0);\n>> +\t\t\t\tsumR += val;\n>> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\tbreak;\n>> +\t\t\tcase 1: /* at Gr pixel */\n>> +\t\t\t\t/* blue: ((0,-1) + (0,1)) / 2 */\n>> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 1;\n>> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* green: (0,0) */\n>> +\t\t\t\tval = *(pin_base + x_0);\n>> +\t\t\t\tsumG += val;\n>> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* red: ((-1,0) + (1,0)) / 2 */\n>> +\t\t\t\tval = ( *(pin_base + x_m1)\n>> +\t\t\t\t\t+ *(pin_base + x_p1) ) >> 1;\n>> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\tbreak;\n>> +\t\t\tcase 2: /* at Gb pixel */\n>> +\t\t\t\t/* blue: ((-1,0) + (1,0)) / 2 */\n>> +\t\t\t\tval = ( *(pin_base + x_m1)\n>> +\t\t\t\t\t+ *(pin_base + x_p1) ) >> 1;\n>> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* green: (0,0) */\n>> +\t\t\t\tval = *(pin_base + x_0);\n>> +\t\t\t\tsumG += val;\n>> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* red: ((0,-1) + (0,1)) / 2 */\n>> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 1;\n>> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\tbreak;\n>> +\t\t\tdefault: /* at B pixel */\n>> +\t\t\t\t/* blue: (0,0) */\n>> +\t\t\t\tval = *(pin_base + x_0);\n>> +\t\t\t\tsumB += val;\n>> +\t\t\t\tval = val * bNumerat_ / bDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* green: ((0,-1)+(-1,0)+(1,0)+(0,1)) / 4 */\n>> +\t\t\t\tval = ( *(pin_base + x_0 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1)\n>> +\t\t\t\t\t+ *(pin_base + x_m1)\n>> +\t\t\t\t\t+ *(pin_base + x_0 + stride_) ) >> 2;\n>> +\t\t\t\tval = val * gNumerat_ / gDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t\t/* red: ((-1,-1)+(1,-1)+(-1,1)+(1,1)) / 4 */\n>> +\t\t\t\tval = ( *(pin_base + x_m1 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1 - stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_m1 + stride_)\n>> +\t\t\t\t\t+ *(pin_base + x_p1 + stride_) ) >> 2;\n>> +\t\t\t\tval = val * rNumerat_ / rDenomin_;\n>> +\t\t\t\t*pout++ = (uint8_t)std::min(val, 0xffU);\n>> +\t\t\t}\n>> +\t\t}\n>> +\t}\n>> +\n>> +\t/* calculate red and blue gains for simple AWB */\n>> +\tLOG(Converter, Debug) << \"sumR = \" << sumR\n>> +\t\t\t << \", sumB = \" << sumB << \", sumG = \" << sumG;\n>> +\n>> +\tsumG /= 2; /* the number of G pixels is twice as big vs R and B ones */\n>> +\n>> +\t/* normalize red, blue, and green sums to fit into 22-bit value */\n>> +\tunsigned long fRed = sumR / 0x400000;\n>> +\tunsigned long fBlue = sumB / 0x400000;\n>> +\tunsigned long fGreen = sumG / 0x400000;\n>> +\tunsigned long fNorm = std::max({ 1UL, fRed, fBlue, fGreen });\n>> +\tsumR /= fNorm;\n>> +\tsumB /= fNorm;\n>> +\tsumG /= fNorm;\n>> +\n>> +\tLOG(Converter, Debug) << \"fNorm = \" << fNorm;\n>> +\tLOG(Converter, Debug) << \"Normalized: sumR = \" << sumR\n>> +\t\t\t << \", sumB= \" << sumB << \", sumG = \" << sumG;\n>> +\n>> +\t/* make sure red/blue gains never exceed approximately 256 */\n>> +\tunsigned long minDenom;\n>> +\trNumerat_ = (sumR + sumB + sumG) / 3;\n>> +\tminDenom = rNumerat_ / 0x100;\n>> +\trDenomin_ = std::max(minDenom, sumR);\n>> +\tbNumerat_ = rNumerat_;\n>> +\tbDenomin_ = std::max(minDenom, sumB);\n>> +\tgNumerat_ = rNumerat_;\n>> +\tgDenomin_ = std::max(minDenom, sumG);\n>> +\n>> +\tLOG(Converter, Debug) << \"rGain = [ \"\n>> +\t\t\t << rNumerat_ << \" / \" << rDenomin_\n>> +\t\t\t << \" ], bGain = [ \" << bNumerat_ << \" / \" << bDenomin_\n>> +\t\t\t << \" ], gGain = [ \" << gNumerat_ << \" / \" << gDenomin_\n>> +\t\t\t << \" (minDenom = \" << minDenom << \")\";\n>> +}\n>> +\n>> +static std::initializer_list compatibles = {};\n>> +\n>> +REGISTER_CONVERTER(\"linaro-sw-converter\", SwConverter, compatibles)\n>> +\n>> +} /* namespace libcamera */\n>> diff --git a/src/libcamera/converter/meson.build b/src/libcamera/converter/meson.build\n>> index 2aa72fe4..1f1e0ea4 100644\n>> --- a/src/libcamera/converter/meson.build\n>> +++ b/src/libcamera/converter/meson.build\n>> @@ -1,5 +1,6 @@\n>> # SPDX-License-Identifier: CC0-1.0\n>>\n>> libcamera_sources += files([\n>> - 'converter_v4l2_m2m.cpp'\n>> + 'converter_softw.cpp',\n>> + 'converter_v4l2_m2m.cpp',\n>> ])\n>> --\n>> 2.34.1\n>>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id AC6E1C326B\n\tfor ;\n\tThu, 21 Sep 2023 19:13:28 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id E459262944;\n\tThu, 21 Sep 2023 21:13:27 +0200 (CEST)","from mail-lf1-x12f.google.com (mail-lf1-x12f.google.com\n\t[IPv6:2a00:1450:4864:20::12f])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 56CF2628D8\n\tfor ;\n\tThu, 21 Sep 2023 21:13:26 +0200 (CEST)","by mail-lf1-x12f.google.com with SMTP id\n\t2adb3069b0e04-5041bb9ce51so2250735e87.1\n\tfor ;\n\tThu, 21 Sep 2023 12:13:26 -0700 (PDT)","from [192.168.118.20] ([87.116.161.81])\n\tby smtp.gmail.com with ESMTPSA id\n\tk2-20020aa7d8c2000000b0052f3051f7d2sm1187280eds.80.2023.09.21.12.13.24\n\t(version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);\n\tThu, 21 Sep 2023 12:13:24 -0700 (PDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1695323607;\n\tbh=Ki9xZawufUE08eVXNhGmu0D8Dk8nHvXazWjXrvOp3Jo=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=H4DmplrDGq+T+klzeA8vnQQn49y4ZsSbFTyY5HLSV9hdov3Ln9IeSas9THl4LQt0Q\n\tMOc1dA3nVcxMaHLF2V1Z++Aj7KWIV/S6hm8b3ItPEWpQasQswDkhwCgAXfKoSLSUZ0\n\twG6xjk2L87rZXLrBIuvzVC9Db+EnzFrHekZ4bhBMNgyj2zC/Ob5+6Tk5J7i/BEMuKO\n\tzJYIWKYCxmpEvmuIsY6YuI2LlDU7raN/2rq+Cy5r59yIaHF89MdAhxIR6I+zz+nJLi\n\tzb8YecbAnb2etboyCxyIhd0Oo7SI5gh2IHbRnV1Nd/23rP2IJerUGUOp6j7Gl2xL0X\n\twQ8Te7fxG3h0w==","v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=linaro.org; s=google; t=1695323605; x=1695928405;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:in-reply-to:from:references:cc:to\n\t:content-language:subject:user-agent:mime-version:date:message-id\n\t:from:to:cc:subject:date:message-id:reply-to;\n\tbh=8cdGysg4CcspGPT8+wIkVVZPr/J+6UFmYzaFQuApDIs=;\n\tb=CA4Hs8zt1x/qE6mK54YTnCtksrDEJYIygeN1AudIWetN713aajJaJgWzp3HXjbR5Po\n\tmDZFivrJlEAa0IUXoxoKHYaT2WZTOvPVgb6WvcJ4l0Bbx+M9CaeY7f0DzzmhoQB1crV/\n\touwDLfTlbe7R3Jk6NMgKYLR8D65RmcHsLy2ISs038Yyz1t8HRE4PhWSIkNqhGGhKLUkb\n\tw+3dLUnDZCA0IOwXcLgpT5OgY1ZZ/btqXsoyGMAFvofj3ZtAgQMcwWa5WlEyeMtHq96K\n\tDWpSR6taCNAdRTBNHRKcULmwjunt+SS7lquIhvN1jC7+oXNtxKKchRm1WGntDSpjYlyD\n\t9gjg=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=linaro.org\n\theader.i=@linaro.org header.b=\"CA4Hs8zt\"; \n\tdkim-atps=neutral","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1695323605; x=1695928405;\n\th=content-transfer-encoding:in-reply-to:from:references:cc:to\n\t:content-language:subject:user-agent:mime-version:date:message-id\n\t:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to;\n\tbh=8cdGysg4CcspGPT8+wIkVVZPr/J+6UFmYzaFQuApDIs=;\n\tb=nng+bqd8K5O1sRWTbtdinevEJZcQAlQSTM0cVzsKitruRPTD8bMlWgnbnKsBQeswiu\n\tCjjJg+1e1/W5ZJtRPDnziehU8NdxqgtJRtc2UXF23Jh+07m5NZ3sDTGkcS4A2OOQP8BT\n\t5M2hT7lngYxu//EiWaOpLTxVJ9jTGIl+VeYj9q45jVDpd0Cba1KOn/qr8AoNVYnbQmWu\n\tsRBRTGgh0gvRW5u78UNglJfxaek8kPXZDTBV3aMFdAm2YZXHmhG8D3CXVOKc4bmdWwHZ\n\tDN/fh+/aHyyOpOfdPKFdkSggEjZvCThZquXr2slZkNkJDW5Mfm8Z7aEZddiIDTP7A66T\n\tH+pw==","X-Gm-Message-State":"AOJu0Yy6rGoGtk+XGMofRqD4NshxUXLn4rhNzdmtQUla2aurqt69xw2N\n\teKbr6iksjtMya327h6qaxSN2xg==","X-Google-Smtp-Source":"AGHT+IHvXWapahUlfxBP79xAmAufPzFtibpmQN3uOrVU0CaEfiWsKFfKi3PdtsMhgAob6MGpuqlAVw==","X-Received":"by 2002:ac2:5f66:0:b0:500:8022:3dc7 with SMTP id\n\tc6-20020ac25f66000000b0050080223dc7mr5754290lfc.10.1695323605248; \n\tThu, 21 Sep 2023 12:13:25 -0700 (PDT)","Message-ID":"","Date":"Thu, 21 Sep 2023 22:13:23 +0300","MIME-Version":"1.0","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n\tThunderbird/102.15.1","Content-Language":"en-US","To":"Jacopo Mondi ","References":"<20230920151921.31273-1-andrey.konovalov@linaro.org>\n\t<20230920151921.31273-4-andrey.konovalov@linaro.org>\n\t","In-Reply-To":"","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","Subject":"Re: [libcamera-devel] [RFC PATCH v2 3/4] libcamera: converter: add\n\tsoftware converter","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Andrey Konovalov via libcamera-devel\n\t","Reply-To":"Andrey Konovalov ","Cc":"bryan.odonoghue@linaro.org, libcamera-devel@lists.libcamera.org,\n\tsrinivas.kandagatla@linaro.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]