[{"id":31354,"web_url":"https://patchwork.libcamera.org/comment/31354/","msgid":"","date":"2024-09-25T08:18:24","subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":148,"url":"https://patchwork.libcamera.org/api/people/148/","name":"Cheng-Hao Yang","email":"chenghaoyang@google.com"},"content":"Friendly ping: I think only 5th and 6th patches need the final reviews\nand stamps, before we can merge this series.\nThanks!\n\nOn Tue, Sep 10, 2024 at 12:48 PM Harvey Yang wrote:\n>\n> Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> loads real images (jpg / jpeg for now) as the source and generates\n> scaled frames.\n>\n> Signed-off-by: Konami Shu \n> Co-developed-by: Harvey Yang \n> Co-developed-by: Yunke Cao \n> Co-developed-by: Tomasz Figa \n> ---\n> .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n> .../pipeline/virtual/image_frame_generator.h | 51 +++++\n> src/libcamera/pipeline/virtual/meson.build | 4 +\n> 3 files changed, 230 insertions(+)\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n>\n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> new file mode 100644\n> index 00000000..80df9d6b\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> @@ -0,0 +1,175 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * image_frame_generator.cpp - Derived class of FrameGenerator for\n> + * generating frames from images\n> + */\n> +\n> +#include \"image_frame_generator.h\"\n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +#include \"libyuv/convert.h\"\n> +#include \"libyuv/scale.h\"\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(Virtual)\n> +\n> +/*\n> + * Factory function to create an ImageFrameGenerator object.\n> + * Read the images and convert them to buffers in NV12 format.\n> + * Store the pointers to the buffers to a list (imageFrameDatas)\n> + */\n> +std::unique_ptr\n> +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> +{\n> + std::unique_ptr imageFrameGenerator =\n> + std::make_unique();\n> + imageFrameGenerator->imageFrames_ = &imageFrames;\n> +\n> + /*\n> + * For each file in the directory, load the image,\n> + * convert it to NV12, and store the pointer.\n> + */\n> + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> + std::filesystem::path path;\n> + if (!imageFrames.number)\n> + /* If the path is to an image */\n> + path = imageFrames.path;\n> + else\n> + /* If the path is to a directory */\n> + path = imageFrames.path / (std::to_string(i) + \".jpg\");\n> +\n> + File file(path);\n> + if (!file.open(File::OpenModeFlag::ReadOnly)) {\n> + LOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> + << \": \" << strerror(file.error());\n> + return nullptr;\n> + }\n> +\n> + /* Read the image file to data */\n> + auto fileSize = file.size();\n> + auto buffer = std::make_unique(file.size());\n> + if (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> + LOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> + << \": \" << strerror(file.error());\n> + return nullptr;\n> + }\n> +\n> + /* Get the width and height of the image */\n> + int width, height;\n> + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> + LOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> + << file.fileName();\n> + return nullptr;\n> + }\n> +\n> + std::unique_ptr dstY =\n> + std::make_unique(width * height);\n> + std::unique_ptr dstUV =\n> + std::make_unique(width * height / 2);\n> + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> + dstY.get(), width, dstUV.get(),\n> + width, width, height, width, height);\n> + if (ret != 0)\n> + LOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> +\n> + imageFrameGenerator->imageFrameDatas_.emplace_back(\n> + ImageFrameData{ std::move(dstY), std::move(dstUV),\n> + Size(width, height) });\n> + }\n> +\n> + return imageFrameGenerator;\n> +}\n> +\n> +/* Scale the buffers for image frames. */\n> +void ImageFrameGenerator::configure(const Size &size)\n> +{\n> + /* Reset the source images to prevent multiple configuration calls */\n> + scaledFrameDatas_.clear();\n> + frameCount_ = 0;\n> + parameter_ = 0;\n> +\n> + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> + /* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> + unsigned int halfSizeWidth = (size.width + 1) / 2;\n> + unsigned int halfSizeHeight = (size.height + 1) / 2;\n> + std::unique_ptr scaledY =\n> + std::make_unique(size.width * size.height);\n> + std::unique_ptr scaledUV =\n> + std::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> + auto &src = imageFrameDatas_[i];\n> +\n> + /*\n> + * \\todo Some platforms might enforce stride due to GPU, like\n> + * ChromeOS ciri (64). The weight needs to be a multiple of\n> + * the stride to work properly for now.\n> + */\n> + libyuv::NV12Scale(src.Y.get(), src.size.width,\n> + src.UV.get(), src.size.width,\n> + src.size.width, src.size.height,\n> + scaledY.get(), size.width, scaledUV.get(), size.width,\n> + size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> +\n> + scaledFrameDatas_.emplace_back(\n> + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> + }\n> +}\n> +\n> +void ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> +{\n> + /* Don't do anything when the list of buffers is empty*/\n> + ASSERT(!scaledFrameDatas_.empty());\n> +\n> + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> +\n> + auto planes = mappedFrameBuffer.planes();\n> +\n> + /* Make sure the frameCount does not over the number of images */\n> + frameCount_ %= imageFrames_->number.value_or(1);\n> +\n> + /* Write the scaledY and scaledUV to the mapped frame buffer */\n> + libyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n> + scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n> + size.width, planes[1].begin(), size.width,\n> + size.width, size.height);\n> +\n> + /* Proceed an image every 4 frames */\n> + /* \\todo Consider setting the proceed interval in the config file */\n> + parameter_++;\n> + if (parameter_ % frameInterval == 0)\n> + frameCount_++;\n> +}\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrameDatas_\n> + * \\brief List of pointers to the not scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::scaledFrameDatas_\n> + * \\brief List of pointers to the scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrames_\n> + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::parameter_\n> + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> + */\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> new file mode 100644\n> index 00000000..bbbe77c8\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> @@ -0,0 +1,51 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * image_frame_generator.h - Derived class of FrameGenerator for\n> + * generating frames from images\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +\n> +#include \"frame_generator.h\"\n> +\n> +namespace libcamera {\n> +\n> +/* Frame configuration provided by the config file */\n> +struct ImageFrames {\n> + std::filesystem::path path;\n> + std::optional number;\n> +};\n> +\n> +class ImageFrameGenerator : public FrameGenerator\n> +{\n> +public:\n> + static std::unique_ptr create(ImageFrames &imageFrames);\n> +\n> +private:\n> + static constexpr unsigned int frameInterval = 4;\n> +\n> + struct ImageFrameData {\n> + std::unique_ptr Y;\n> + std::unique_ptr UV;\n> + Size size;\n> + };\n> +\n> + void configure(const Size &size) override;\n> + void generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> +\n> + std::vector imageFrameDatas_;\n> + std::vector scaledFrameDatas_;\n> + ImageFrames *imageFrames_;\n> + unsigned int frameCount_;\n> + unsigned int parameter_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> index 0c537777..73d10cc3 100644\n> --- a/src/libcamera/pipeline/virtual/meson.build\n> +++ b/src/libcamera/pipeline/virtual/meson.build\n> @@ -1,8 +1,12 @@\n> # SPDX-License-Identifier: CC0-1.0\n>\n> libcamera_internal_sources += files([\n> + 'image_frame_generator.cpp',\n> 'test_pattern_generator.cpp',\n> 'virtual.cpp',\n> ])\n>\n> +libjpeg = dependency('libjpeg', required : false)\n> +\n> libcamera_deps += [libyuv_dep]\n> +libcamera_deps += [libjpeg]\n> --\n> 2.46.0.598.g6f2099f65c-goog\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 2777CC3257\n\tfor ;\n\tWed, 25 Sep 2024 08:19:03 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6AB926350F;\n\tWed, 25 Sep 2024 10:19:02 +0200 (CEST)","from mail-ed1-x52d.google.com (mail-ed1-x52d.google.com\n\t[IPv6:2a00:1450:4864:20::52d])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id AA051634F4\n\tfor ;\n\tWed, 25 Sep 2024 10:19:01 +0200 (CEST)","by mail-ed1-x52d.google.com with SMTP id\n\t4fb4d7f45d1cf-5c2443b2581so22428a12.0\n\tfor ;\n\tWed, 25 Sep 2024 01:19:01 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=google.com header.i=@google.com\n\theader.b=\"CyZuAyx5\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=google.com; s=20230601; t=1727252341; x=1727857141;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=nfQQOjviRiV55wWTlf7xJwZWddh3djvnrlXuaOMHft4=;\n\tb=CyZuAyx5DUFFoQ2C1D9H3ilCCuMNeDhecRuGXUcg8WpbemdAlqc5mf9l8BXtLdvvhu\n\trqpZohvbIel7h3RezWhycJvkUGx0b82NAp6Qds1iNuHtwi+ofb6FiAfKnkQtPneT0WvX\n\tV4j5YwMGHUo3Lvn+kEZK2BM8rmnooNDI/bNpoondyUswYIlZerylBP8I0bWwWsrOhdq+\n\tg8eBByaaE2yVeJLUNEIiyJ6zTlItEBEaITem8dNn71T9mKgqqfVyV76p9glePthsvZ4Z\n\tPGXX20m/lBgzBG1yE9zNNy1XROWqzoLxs3RjzPQ+F/QeJt04R3MP5jRdhADmJ5eZit05\n\tYJXQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1727252341; x=1727857141;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=nfQQOjviRiV55wWTlf7xJwZWddh3djvnrlXuaOMHft4=;\n\tb=wNqEZGHo+WzgrVV4wn8vDee5E8FYrn8fvzYbDDBOFhX7i/nCsdL3Ye2Jv2mTZeqFJC\n\tFATuVJ4v8d22TwQVI9XTF7Wur4vWu6O56z+IpNQj79IfgGG8blAcJBvfW5xhSW894p3p\n\t1ebniOC7ZW5ijvSD0Loa5aBfbZEPpsjyhgrFyJJqv4q8d+h9aXdguskMe0VzXcKFVZ2Y\n\tVbeRzyDf0rB7YKUJpAIOrzouIrzvYpNeE+M0POP1TB3tCgaEipqKmajmFNEyDDJ0fRkP\n\ti9uMQi1GZmNRtN0DJ+Ma3rjfmT1mai7sL4tFP87LAFGahq4rYB7rK+o2BtFqjwaY/cuz\n\tnBRA==","X-Gm-Message-State":"AOJu0YyyBkCXfWTCMOo5Q8WK5C5NatFcCHkohjJtF70QnO9I/sJLhZLa\n\tVYX9bB0EfEF02cV2tIhyqY9AIar+popINOOit1URP1BJYY3HpuPLutl7BbF6VqcAlLUMj1tEl2m\n\taOZkc22cjdScAz6bkPTttXDs/ucjaQEE22jDh","X-Google-Smtp-Source":"AGHT+IF7g4DXR6BntnfUA/Z8olHttx7qtw4hJKNOd7GJ6+kp73bb92g/ig45Aaw04SaJyDQiVCVLE3+PgcfAfWmlOTU=","X-Received":"by 2002:a05:6402:2794:b0:5c5:c44d:484e with SMTP id\n\t4fb4d7f45d1cf-5c720f8f08bmr263476a12.1.1727252340816; Wed, 25 Sep 2024\n\t01:19:00 -0700 (PDT)","MIME-Version":"1.0","References":"<20240910044834.2477701-1-chenghaoyang@google.com>\n\t<20240910044834.2477701-6-chenghaoyang@google.com>","In-Reply-To":"<20240910044834.2477701-6-chenghaoyang@google.com>","From":"Cheng-Hao Yang ","Date":"Wed, 25 Sep 2024 16:18:24 +0800","Message-ID":"","Subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","To":"Harvey Yang ","Cc":"libcamera-devel@lists.libcamera.org, Konami Shu , \n\tYunke Cao , Tomasz Figa ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":31391,"web_url":"https://patchwork.libcamera.org/comment/31391/","msgid":"<47itxygvnce27mxaqmlyfe7kyzieohzdyu7vztquatmjttklur@66rawijmlq5d>","date":"2024-09-26T10:17:13","subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Wed, Sep 25, 2024 at 04:18:24PM GMT, Cheng-Hao Yang wrote:\n> Friendly ping: I think only 5th and 6th patches need the final reviews\n> and stamps, before we can merge this series.\n> Thanks!\n\nI still see two failing tests\n11/75 libcamera:gstreamer / single_stream_test FAIL 2.41s (exit status 255 or signal 127 SIGinvalid)\n14/75 libcamera:gstreamer / memory_lifetime_test FAIL 0.04s (exit status 255 or signal 127 SIGinvalid)\n\nThey both fail with:\nUnable to create pipeline (nil).0xa676a60\n\n>\n> On Tue, Sep 10, 2024 at 12:48 PM Harvey Yang wrote:\n> >\n> > Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> > loads real images (jpg / jpeg for now) as the source and generates\n> > scaled frames.\n> >\n> > Signed-off-by: Konami Shu \n> > Co-developed-by: Harvey Yang \n> > Co-developed-by: Yunke Cao \n> > Co-developed-by: Tomasz Figa \n> > ---\n> > .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n> > .../pipeline/virtual/image_frame_generator.h | 51 +++++\n> > src/libcamera/pipeline/virtual/meson.build | 4 +\n> > 3 files changed, 230 insertions(+)\n> > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n> >\n> > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > new file mode 100644\n> > index 00000000..80df9d6b\n> > --- /dev/null\n> > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > @@ -0,0 +1,175 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2024, Google Inc.\n> > + *\n> > + * image_frame_generator.cpp - Derived class of FrameGenerator for\n> > + * generating frames from images\n> > + */\n> > +\n> > +#include \"image_frame_generator.h\"\n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +\n> > +#include \"libcamera/internal/mapped_framebuffer.h\"\n> > +\n> > +#include \"libyuv/convert.h\"\n> > +#include \"libyuv/scale.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +LOG_DECLARE_CATEGORY(Virtual)\n> > +\n> > +/*\n> > + * Factory function to create an ImageFrameGenerator object.\n> > + * Read the images and convert them to buffers in NV12 format.\n> > + * Store the pointers to the buffers to a list (imageFrameDatas)\n> > + */\n> > +std::unique_ptr\n> > +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> > +{\n> > + std::unique_ptr imageFrameGenerator =\n> > + std::make_unique();\n> > + imageFrameGenerator->imageFrames_ = &imageFrames;\n> > +\n> > + /*\n> > + * For each file in the directory, load the image,\n> > + * convert it to NV12, and store the pointer.\n> > + */\n> > + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> > + std::filesystem::path path;\n> > + if (!imageFrames.number)\n> > + /* If the path is to an image */\n> > + path = imageFrames.path;\n> > + else\n> > + /* If the path is to a directory */\n> > + path = imageFrames.path / (std::to_string(i) + \".jpg\");\n> > +\n> > + File file(path);\n> > + if (!file.open(File::OpenModeFlag::ReadOnly)) {\n> > + LOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> > + << \": \" << strerror(file.error());\n> > + return nullptr;\n> > + }\n> > +\n> > + /* Read the image file to data */\n> > + auto fileSize = file.size();\n> > + auto buffer = std::make_unique(file.size());\n> > + if (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> > + LOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> > + << \": \" << strerror(file.error());\n> > + return nullptr;\n> > + }\n> > +\n> > + /* Get the width and height of the image */\n> > + int width, height;\n> > + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> > + LOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> > + << file.fileName();\n> > + return nullptr;\n> > + }\n> > +\n> > + std::unique_ptr dstY =\n> > + std::make_unique(width * height);\n> > + std::unique_ptr dstUV =\n> > + std::make_unique(width * height / 2);\n> > + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> > + dstY.get(), width, dstUV.get(),\n> > + width, width, height, width, height);\n> > + if (ret != 0)\n> > + LOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> > +\n> > + imageFrameGenerator->imageFrameDatas_.emplace_back(\n> > + ImageFrameData{ std::move(dstY), std::move(dstUV),\n> > + Size(width, height) });\n> > + }\n> > +\n> > + return imageFrameGenerator;\n> > +}\n> > +\n> > +/* Scale the buffers for image frames. */\n> > +void ImageFrameGenerator::configure(const Size &size)\n> > +{\n> > + /* Reset the source images to prevent multiple configuration calls */\n> > + scaledFrameDatas_.clear();\n> > + frameCount_ = 0;\n> > + parameter_ = 0;\n> > +\n> > + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> > + /* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> > + unsigned int halfSizeWidth = (size.width + 1) / 2;\n> > + unsigned int halfSizeHeight = (size.height + 1) / 2;\n> > + std::unique_ptr scaledY =\n> > + std::make_unique(size.width * size.height);\n> > + std::unique_ptr scaledUV =\n> > + std::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> > + auto &src = imageFrameDatas_[i];\n> > +\n> > + /*\n> > + * \\todo Some platforms might enforce stride due to GPU, like\n> > + * ChromeOS ciri (64). The weight needs to be a multiple of\n> > + * the stride to work properly for now.\n> > + */\n> > + libyuv::NV12Scale(src.Y.get(), src.size.width,\n> > + src.UV.get(), src.size.width,\n> > + src.size.width, src.size.height,\n> > + scaledY.get(), size.width, scaledUV.get(), size.width,\n> > + size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> > +\n> > + scaledFrameDatas_.emplace_back(\n> > + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> > + }\n> > +}\n> > +\n> > +void ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> > +{\n> > + /* Don't do anything when the list of buffers is empty*/\n> > + ASSERT(!scaledFrameDatas_.empty());\n> > +\n> > + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> > +\n> > + auto planes = mappedFrameBuffer.planes();\n> > +\n> > + /* Make sure the frameCount does not over the number of images */\n> > + frameCount_ %= imageFrames_->number.value_or(1);\n> > +\n> > + /* Write the scaledY and scaledUV to the mapped frame buffer */\n> > + libyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n> > + scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n> > + size.width, planes[1].begin(), size.width,\n> > + size.width, size.height);\n> > +\n> > + /* Proceed an image every 4 frames */\n> > + /* \\todo Consider setting the proceed interval in the config file */\n> > + parameter_++;\n> > + if (parameter_ % frameInterval == 0)\n> > + frameCount_++;\n> > +}\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::imageFrameDatas_\n> > + * \\brief List of pointers to the not scaled image buffers\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::scaledFrameDatas_\n> > + * \\brief List of pointers to the scaled image buffers\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::imageFrames_\n> > + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::parameter_\n> > + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> > + */\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > new file mode 100644\n> > index 00000000..bbbe77c8\n> > --- /dev/null\n> > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > @@ -0,0 +1,51 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2024, Google Inc.\n> > + *\n> > + * image_frame_generator.h - Derived class of FrameGenerator for\n> > + * generating frames from images\n> > + */\n> > +\n> > +#pragma once\n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \"frame_generator.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +/* Frame configuration provided by the config file */\n> > +struct ImageFrames {\n> > + std::filesystem::path path;\n> > + std::optional number;\n> > +};\n> > +\n> > +class ImageFrameGenerator : public FrameGenerator\n> > +{\n> > +public:\n> > + static std::unique_ptr create(ImageFrames &imageFrames);\n> > +\n> > +private:\n> > + static constexpr unsigned int frameInterval = 4;\n> > +\n> > + struct ImageFrameData {\n> > + std::unique_ptr Y;\n> > + std::unique_ptr UV;\n> > + Size size;\n> > + };\n> > +\n> > + void configure(const Size &size) override;\n> > + void generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> > +\n> > + std::vector imageFrameDatas_;\n> > + std::vector scaledFrameDatas_;\n> > + ImageFrames *imageFrames_;\n> > + unsigned int frameCount_;\n> > + unsigned int parameter_;\n> > +};\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> > index 0c537777..73d10cc3 100644\n> > --- a/src/libcamera/pipeline/virtual/meson.build\n> > +++ b/src/libcamera/pipeline/virtual/meson.build\n> > @@ -1,8 +1,12 @@\n> > # SPDX-License-Identifier: CC0-1.0\n> >\n> > libcamera_internal_sources += files([\n> > + 'image_frame_generator.cpp',\n> > 'test_pattern_generator.cpp',\n> > 'virtual.cpp',\n> > ])\n> >\n> > +libjpeg = dependency('libjpeg', required : false)\n> > +\n> > libcamera_deps += [libyuv_dep]\n> > +libcamera_deps += [libjpeg]\n> > --\n> > 2.46.0.598.g6f2099f65c-goog\n> >\n>\n>\n> --\n> BR,\n> Harvey Yang","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 896F9C3257\n\tfor ;\n\tThu, 26 Sep 2024 10:17:19 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id D75876350F;\n\tThu, 26 Sep 2024 12:17:17 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id EAC91634F9\n\tfor ;\n\tThu, 26 Sep 2024 12:17:15 +0200 (CEST)","from ideasonboard.com (mob-5-90-51-229.net.vodafone.it\n\t[5.90.51.229])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A919B169;\n\tThu, 26 Sep 2024 12:15:47 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"UF0373XQ\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1727345747;\n\tbh=O5rKx4VsmaQ8bRCoWzxXrzL8WT8FRVvJeZ7rslDqS6Y=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=UF0373XQLIPeWhXBzPIZf3Q0FTbeivRQ/5G3V2lyDJv/cot31PzywGdoGAdqsLi8i\n\t+6kFGC5u4vdCc9PBA6P4nCcq/m4iPJP8zF4UA7x6YBGeuqarB/wG5+2KcM3CYpotMD\n\tkSXZf9LTqUWmhggmyzE6hax5pG0WdH5DvLrXbgcI=","Date":"Thu, 26 Sep 2024 12:17:13 +0200","From":"Jacopo Mondi ","To":"Cheng-Hao Yang ","Cc":"Harvey Yang , \n\tlibcamera-devel@lists.libcamera.org, Konami Shu , \n\tYunke Cao , Tomasz Figa ","Subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","Message-ID":"<47itxygvnce27mxaqmlyfe7kyzieohzdyu7vztquatmjttklur@66rawijmlq5d>","References":"<20240910044834.2477701-1-chenghaoyang@google.com>\n\t<20240910044834.2477701-6-chenghaoyang@google.com>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","Content-Transfer-Encoding":"8bit","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":31401,"web_url":"https://patchwork.libcamera.org/comment/31401/","msgid":"<2m37vv3nlor2ajoa6snibsv3i5k7gjnmla5caxfpoammxxuwaj@awmtffif3mal>","date":"2024-09-26T11:30:53","subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Harvey\n\nOn Tue, Sep 10, 2024 at 04:40:18AM GMT, Harvey Yang wrote:\n> Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> loads real images (jpg / jpeg for now) as the source and generates\n> scaled frames.\n>\n> Signed-off-by: Konami Shu \n> Co-developed-by: Harvey Yang \n> Co-developed-by: Yunke Cao \n> Co-developed-by: Tomasz Figa \n> ---\n> .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n> .../pipeline/virtual/image_frame_generator.h | 51 +++++\n> src/libcamera/pipeline/virtual/meson.build | 4 +\n> 3 files changed, 230 insertions(+)\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n>\n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> new file mode 100644\n> index 00000000..80df9d6b\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> @@ -0,0 +1,175 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * image_frame_generator.cpp - Derived class of FrameGenerator for\n> + * generating frames from images\n> + */\n> +\n> +#include \"image_frame_generator.h\"\n> +\n> +#include \n> +#include \n\nBoth already included in the header file\n\n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +#include \"libyuv/convert.h\"\n> +#include \"libyuv/scale.h\"\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(Virtual)\n> +\n> +/*\n> + * Factory function to create an ImageFrameGenerator object.\n> + * Read the images and convert them to buffers in NV12 format.\n> + * Store the pointers to the buffers to a list (imageFrameDatas)\n> + */\n> +std::unique_ptr\n> +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> +{\n> +\tstd::unique_ptr imageFrameGenerator =\n> +\t\tstd::make_unique();\n> +\timageFrameGenerator->imageFrames_ = &imageFrames;\n> +\n> +\t/*\n> +\t * For each file in the directory, load the image,\n> +\t * convert it to NV12, and store the pointer.\n> +\t */\n> +\tfor (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> +\t\tstd::filesystem::path path;\n> +\t\tif (!imageFrames.number)\n> +\t\t\t/* If the path is to an image */\n> +\t\t\tpath = imageFrames.path;\n> +\t\telse\n> +\t\t\t/* If the path is to a directory */\n> +\t\t\tpath = imageFrames.path / (std::to_string(i) + \".jpg\");\n> +\n> +\t\tFile file(path);\n> +\t\tif (!file.open(File::OpenModeFlag::ReadOnly)) {\n> +\t\t\tLOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> +\t\t\t\t\t << \": \" << strerror(file.error());\n> +\t\t\treturn nullptr;\n> +\t\t}\n> +\n> +\t\t/* Read the image file to data */\n> +\t\tauto fileSize = file.size();\n> +\t\tauto buffer = std::make_unique(file.size());\n> +\t\tif (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> +\t\t\tLOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> +\t\t\t\t\t << \": \" << strerror(file.error());\n> +\t\t\treturn nullptr;\n> +\t\t}\n> +\n> +\t\t/* Get the width and height of the image */\n> +\t\tint width, height;\n> +\t\tif (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> +\t\t\tLOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> +\t\t\t\t\t << file.fileName();\n> +\t\t\treturn nullptr;\n> +\t\t}\n> +\n> +\t\tstd::unique_ptr dstY =\n> +\t\t\tstd::make_unique(width * height);\n> +\t\tstd::unique_ptr dstUV =\n> +\t\t\tstd::make_unique(width * height / 2);\n> +\t\tint ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> +\t\t\t\t\t dstY.get(), width, dstUV.get(),\n> +\t\t\t\t\t width, width, height, width, height);\n> +\t\tif (ret != 0)\n> +\t\t\tLOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> +\n> +\t\timageFrameGenerator->imageFrameDatas_.emplace_back(\n> +\t\t\tImageFrameData{ std::move(dstY), std::move(dstUV),\n> +\t\t\t\t\tSize(width, height) });\n> +\t}\n> +\n> +\treturn imageFrameGenerator;\n> +}\n> +\n> +/* Scale the buffers for image frames. */\n> +void ImageFrameGenerator::configure(const Size &size)\n> +{\n> +\t/* Reset the source images to prevent multiple configuration calls */\n> +\tscaledFrameDatas_.clear();\n> +\tframeCount_ = 0;\n> +\tparameter_ = 0;\n> +\n> +\tfor (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> +\t\t/* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> +\t\tunsigned int halfSizeWidth = (size.width + 1) / 2;\n> +\t\tunsigned int halfSizeHeight = (size.height + 1) / 2;\n> +\t\tstd::unique_ptr scaledY =\n> +\t\t\tstd::make_unique(size.width * size.height);\n> +\t\tstd::unique_ptr scaledUV =\n> +\t\t\tstd::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> +\t\tauto &src = imageFrameDatas_[i];\n> +\n> +\t\t/*\n> +\t\t * \\todo Some platforms might enforce stride due to GPU, like\n> +\t\t * ChromeOS ciri (64). The weight needs to be a multiple of\n> +\t\t * the stride to work properly for now.\n> +\t\t */\n> +\t\tlibyuv::NV12Scale(src.Y.get(), src.size.width,\n> +\t\t\t\t src.UV.get(), src.size.width,\n> +\t\t\t\t src.size.width, src.size.height,\n> +\t\t\t\t scaledY.get(), size.width, scaledUV.get(), size.width,\n> +\t\t\t\t size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> +\n> +\t\tscaledFrameDatas_.emplace_back(\n> +\t\t\tImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> +\t}\n> +}\n> +\n> +void ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> +{\n> +\t/* Don't do anything when the list of buffers is empty*/\n> +\tASSERT(!scaledFrameDatas_.empty());\n> +\n> +\tMappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> +\n> +\tauto planes = mappedFrameBuffer.planes();\n> +\n> +\t/* Make sure the frameCount does not over the number of images */\n> +\tframeCount_ %= imageFrames_->number.value_or(1);\n> +\n> +\t/* Write the scaledY and scaledUV to the mapped frame buffer */\n> +\tlibyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n> +\t\t\t scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n> +\t\t\t size.width, planes[1].begin(), size.width,\n> +\t\t\t size.width, size.height);\n> +\n> +\t/* Proceed an image every 4 frames */\n> +\t/* \\todo Consider setting the proceed interval in the config file */\n> +\tparameter_++;\n> +\tif (parameter_ % frameInterval == 0)\n> +\t\tframeCount_++;\n> +}\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrameDatas_\n> + * \\brief List of pointers to the not scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::scaledFrameDatas_\n> + * \\brief List of pointers to the scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrames_\n> + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::parameter_\n> + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> + */\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> new file mode 100644\n> index 00000000..bbbe77c8\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> @@ -0,0 +1,51 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * image_frame_generator.h - Derived class of FrameGenerator for\n> + * generating frames from images\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +\n> +#include \"frame_generator.h\"\n> +\n> +namespace libcamera {\n> +\n> +/* Frame configuration provided by the config file */\n> +struct ImageFrames {\n> +\tstd::filesystem::path path;\n> +\tstd::optional number;\n> +};\n> +\n> +class ImageFrameGenerator : public FrameGenerator\n> +{\n> +public:\n> +\tstatic std::unique_ptr create(ImageFrames &imageFrames);\n> +\n> +private:\n> +\tstatic constexpr unsigned int frameInterval = 4;\n\nmaybe kFrameInterval\n\n> +\n> +\tstruct ImageFrameData {\n> +\t\tstd::unique_ptr Y;\n> +\t\tstd::unique_ptr UV;\n> +\t\tSize size;\n> +\t};\n> +\n> +\tvoid configure(const Size &size) override;\n> +\tvoid generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> +\n> +\tstd::vector imageFrameDatas_;\n> +\tstd::vector scaledFrameDatas_;\n> +\tImageFrames *imageFrames_;\n> +\tunsigned int frameCount_;\n> +\tunsigned int parameter_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> index 0c537777..73d10cc3 100644\n> --- a/src/libcamera/pipeline/virtual/meson.build\n> +++ b/src/libcamera/pipeline/virtual/meson.build\n> @@ -1,8 +1,12 @@\n> # SPDX-License-Identifier: CC0-1.0\n>\n> libcamera_internal_sources += files([\n> + 'image_frame_generator.cpp',\n> 'test_pattern_generator.cpp',\n> 'virtual.cpp',\n> ])\n>\n> +libjpeg = dependency('libjpeg', required : false)\n> +\n\nIsn't this required now ?\n\nThe rest looks good, thanks!\n\nwith the above nits fixed\nReviewed-by: Jacopo Mondi \n\nThanks\n j\n\n\n> libcamera_deps += [libyuv_dep]\n> +libcamera_deps += [libjpeg]\n> --\n> 2.46.0.598.g6f2099f65c-goog\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D3BF4C3257\n\tfor ;\n\tThu, 26 Sep 2024 11:30:59 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C451B6350F;\n\tThu, 26 Sep 2024 13:30:58 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id E53EE634F9\n\tfor ;\n\tThu, 26 Sep 2024 13:30:56 +0200 (CEST)","from ideasonboard.com (mob-5-90-51-229.net.vodafone.it\n\t[5.90.51.229])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 9427B8D4;\n\tThu, 26 Sep 2024 13:29:28 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"Wpd1lu3T\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1727350168;\n\tbh=dXlqLIFXOmDXYsOpxdFaSxIbZallsa3OsqhO+ByHa6Q=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=Wpd1lu3TU3WXGvyR6E5KUjoQ4MQca+W4TqckfBwxYWdi9MWQBN5/ZyVbUYR7NLcU6\n\tYJ3iT5mMuC62iygb9CrHoRl80MFgI3mVIEBUM66kBEsKp3zQoW+9m0FssWhBFMIQP4\n\t+p3gG5RpsaEHbhionbPW4WE0wtSXnxHJWi8xNcNg=","Date":"Thu, 26 Sep 2024 13:30:53 +0200","From":"Jacopo Mondi ","To":"Harvey Yang ","Cc":"libcamera-devel@lists.libcamera.org, \n\tHarvey Yang , Konami Shu ,\n\tYunke Cao , Tomasz Figa ","Subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","Message-ID":"<2m37vv3nlor2ajoa6snibsv3i5k7gjnmla5caxfpoammxxuwaj@awmtffif3mal>","References":"<20240910044834.2477701-1-chenghaoyang@google.com>\n\t<20240910044834.2477701-6-chenghaoyang@google.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20240910044834.2477701-6-chenghaoyang@google.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":31416,"web_url":"https://patchwork.libcamera.org/comment/31416/","msgid":"","date":"2024-09-26T16:32:26","subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":148,"url":"https://patchwork.libcamera.org/api/people/148/","name":"Cheng-Hao Yang","email":"chenghaoyang@google.com"},"content":"Hi Jacopo,\n\nOn Thu, Sep 26, 2024 at 6:17 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Wed, Sep 25, 2024 at 04:18:24PM GMT, Cheng-Hao Yang wrote:\n> > Friendly ping: I think only 5th and 6th patches need the final reviews\n> > and stamps, before we can merge this series.\n> > Thanks!\n>\n> I still see two failing tests\n> 11/75 libcamera:gstreamer / single_stream_test FAIL 2.41s (exit status 255 or signal 127 SIGinvalid)\n> 14/75 libcamera:gstreamer / memory_lifetime_test FAIL 0.04s (exit status 255 or signal 127 SIGinvalid)\n>\n> They both fail with:\n> Unable to create pipeline (nil).0xa676a60\n\nHow did you run the tests?\nThe gitlab pipeline passes those tests though:\nhttps://gitlab.freedesktop.org/chenghaoyang/libcamera/-/jobs/64229804\n\n\n\n>\n> >\n> > On Tue, Sep 10, 2024 at 12:48 PM Harvey Yang wrote:\n> > >\n> > > Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> > > loads real images (jpg / jpeg for now) as the source and generates\n> > > scaled frames.\n> > >\n> > > Signed-off-by: Konami Shu \n> > > Co-developed-by: Harvey Yang \n> > > Co-developed-by: Yunke Cao \n> > > Co-developed-by: Tomasz Figa \n> > > ---\n> > > .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n> > > .../pipeline/virtual/image_frame_generator.h | 51 +++++\n> > > src/libcamera/pipeline/virtual/meson.build | 4 +\n> > > 3 files changed, 230 insertions(+)\n> > > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n> > >\n> > > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > > new file mode 100644\n> > > index 00000000..80df9d6b\n> > > --- /dev/null\n> > > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > > @@ -0,0 +1,175 @@\n> > > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > > +/*\n> > > + * Copyright (C) 2024, Google Inc.\n> > > + *\n> > > + * image_frame_generator.cpp - Derived class of FrameGenerator for\n> > > + * generating frames from images\n> > > + */\n> > > +\n> > > +#include \"image_frame_generator.h\"\n> > > +\n> > > +#include \n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \n> > > +\n> > > +#include \"libcamera/internal/mapped_framebuffer.h\"\n> > > +\n> > > +#include \"libyuv/convert.h\"\n> > > +#include \"libyuv/scale.h\"\n> > > +\n> > > +namespace libcamera {\n> > > +\n> > > +LOG_DECLARE_CATEGORY(Virtual)\n> > > +\n> > > +/*\n> > > + * Factory function to create an ImageFrameGenerator object.\n> > > + * Read the images and convert them to buffers in NV12 format.\n> > > + * Store the pointers to the buffers to a list (imageFrameDatas)\n> > > + */\n> > > +std::unique_ptr\n> > > +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> > > +{\n> > > + std::unique_ptr imageFrameGenerator =\n> > > + std::make_unique();\n> > > + imageFrameGenerator->imageFrames_ = &imageFrames;\n> > > +\n> > > + /*\n> > > + * For each file in the directory, load the image,\n> > > + * convert it to NV12, and store the pointer.\n> > > + */\n> > > + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> > > + std::filesystem::path path;\n> > > + if (!imageFrames.number)\n> > > + /* If the path is to an image */\n> > > + path = imageFrames.path;\n> > > + else\n> > > + /* If the path is to a directory */\n> > > + path = imageFrames.path / (std::to_string(i) + \".jpg\");\n> > > +\n> > > + File file(path);\n> > > + if (!file.open(File::OpenModeFlag::ReadOnly)) {\n> > > + LOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> > > + << \": \" << strerror(file.error());\n> > > + return nullptr;\n> > > + }\n> > > +\n> > > + /* Read the image file to data */\n> > > + auto fileSize = file.size();\n> > > + auto buffer = std::make_unique(file.size());\n> > > + if (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> > > + LOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> > > + << \": \" << strerror(file.error());\n> > > + return nullptr;\n> > > + }\n> > > +\n> > > + /* Get the width and height of the image */\n> > > + int width, height;\n> > > + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> > > + LOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> > > + << file.fileName();\n> > > + return nullptr;\n> > > + }\n> > > +\n> > > + std::unique_ptr dstY =\n> > > + std::make_unique(width * height);\n> > > + std::unique_ptr dstUV =\n> > > + std::make_unique(width * height / 2);\n> > > + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> > > + dstY.get(), width, dstUV.get(),\n> > > + width, width, height, width, height);\n> > > + if (ret != 0)\n> > > + LOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> > > +\n> > > + imageFrameGenerator->imageFrameDatas_.emplace_back(\n> > > + ImageFrameData{ std::move(dstY), std::move(dstUV),\n> > > + Size(width, height) });\n> > > + }\n> > > +\n> > > + return imageFrameGenerator;\n> > > +}\n> > > +\n> > > +/* Scale the buffers for image frames. */\n> > > +void ImageFrameGenerator::configure(const Size &size)\n> > > +{\n> > > + /* Reset the source images to prevent multiple configuration calls */\n> > > + scaledFrameDatas_.clear();\n> > > + frameCount_ = 0;\n> > > + parameter_ = 0;\n> > > +\n> > > + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> > > + /* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> > > + unsigned int halfSizeWidth = (size.width + 1) / 2;\n> > > + unsigned int halfSizeHeight = (size.height + 1) / 2;\n> > > + std::unique_ptr scaledY =\n> > > + std::make_unique(size.width * size.height);\n> > > + std::unique_ptr scaledUV =\n> > > + std::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> > > + auto &src = imageFrameDatas_[i];\n> > > +\n> > > + /*\n> > > + * \\todo Some platforms might enforce stride due to GPU, like\n> > > + * ChromeOS ciri (64). The weight needs to be a multiple of\n> > > + * the stride to work properly for now.\n> > > + */\n> > > + libyuv::NV12Scale(src.Y.get(), src.size.width,\n> > > + src.UV.get(), src.size.width,\n> > > + src.size.width, src.size.height,\n> > > + scaledY.get(), size.width, scaledUV.get(), size.width,\n> > > + size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> > > +\n> > > + scaledFrameDatas_.emplace_back(\n> > > + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> > > + }\n> > > +}\n> > > +\n> > > +void ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> > > +{\n> > > + /* Don't do anything when the list of buffers is empty*/\n> > > + ASSERT(!scaledFrameDatas_.empty());\n> > > +\n> > > + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> > > +\n> > > + auto planes = mappedFrameBuffer.planes();\n> > > +\n> > > + /* Make sure the frameCount does not over the number of images */\n> > > + frameCount_ %= imageFrames_->number.value_or(1);\n> > > +\n> > > + /* Write the scaledY and scaledUV to the mapped frame buffer */\n> > > + libyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n> > > + scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n> > > + size.width, planes[1].begin(), size.width,\n> > > + size.width, size.height);\n> > > +\n> > > + /* Proceed an image every 4 frames */\n> > > + /* \\todo Consider setting the proceed interval in the config file */\n> > > + parameter_++;\n> > > + if (parameter_ % frameInterval == 0)\n> > > + frameCount_++;\n> > > +}\n> > > +\n> > > +/*\n> > > + * \\var ImageFrameGenerator::imageFrameDatas_\n> > > + * \\brief List of pointers to the not scaled image buffers\n> > > + */\n> > > +\n> > > +/*\n> > > + * \\var ImageFrameGenerator::scaledFrameDatas_\n> > > + * \\brief List of pointers to the scaled image buffers\n> > > + */\n> > > +\n> > > +/*\n> > > + * \\var ImageFrameGenerator::imageFrames_\n> > > + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> > > + */\n> > > +\n> > > +/*\n> > > + * \\var ImageFrameGenerator::parameter_\n> > > + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> > > + */\n> > > +\n> > > +} /* namespace libcamera */\n> > > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > > new file mode 100644\n> > > index 00000000..bbbe77c8\n> > > --- /dev/null\n> > > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > > @@ -0,0 +1,51 @@\n> > > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > > +/*\n> > > + * Copyright (C) 2024, Google Inc.\n> > > + *\n> > > + * image_frame_generator.h - Derived class of FrameGenerator for\n> > > + * generating frames from images\n> > > + */\n> > > +\n> > > +#pragma once\n> > > +\n> > > +#include \n> > > +#include \n> > > +#include \n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \"frame_generator.h\"\n> > > +\n> > > +namespace libcamera {\n> > > +\n> > > +/* Frame configuration provided by the config file */\n> > > +struct ImageFrames {\n> > > + std::filesystem::path path;\n> > > + std::optional number;\n> > > +};\n> > > +\n> > > +class ImageFrameGenerator : public FrameGenerator\n> > > +{\n> > > +public:\n> > > + static std::unique_ptr create(ImageFrames &imageFrames);\n> > > +\n> > > +private:\n> > > + static constexpr unsigned int frameInterval = 4;\n> > > +\n> > > + struct ImageFrameData {\n> > > + std::unique_ptr Y;\n> > > + std::unique_ptr UV;\n> > > + Size size;\n> > > + };\n> > > +\n> > > + void configure(const Size &size) override;\n> > > + void generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> > > +\n> > > + std::vector imageFrameDatas_;\n> > > + std::vector scaledFrameDatas_;\n> > > + ImageFrames *imageFrames_;\n> > > + unsigned int frameCount_;\n> > > + unsigned int parameter_;\n> > > +};\n> > > +\n> > > +} /* namespace libcamera */\n> > > diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> > > index 0c537777..73d10cc3 100644\n> > > --- a/src/libcamera/pipeline/virtual/meson.build\n> > > +++ b/src/libcamera/pipeline/virtual/meson.build\n> > > @@ -1,8 +1,12 @@\n> > > # SPDX-License-Identifier: CC0-1.0\n> > >\n> > > libcamera_internal_sources += files([\n> > > + 'image_frame_generator.cpp',\n> > > 'test_pattern_generator.cpp',\n> > > 'virtual.cpp',\n> > > ])\n> > >\n> > > +libjpeg = dependency('libjpeg', required : false)\n> > > +\n> > > libcamera_deps += [libyuv_dep]\n> > > +libcamera_deps += [libjpeg]\n> > > --\n> > > 2.46.0.598.g6f2099f65c-goog\n> > >\n> >\n> >\n> > --\n> > BR,\n> > Harvey Yang\n\n\n\n--\nBR,\nHarvey Yang","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 5DF07C0F1B\n\tfor ;\n\tThu, 26 Sep 2024 16:33:05 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1298963520;\n\tThu, 26 Sep 2024 18:33:05 +0200 (CEST)","from mail-lf1-x135.google.com (mail-lf1-x135.google.com\n\t[IPv6:2a00:1450:4864:20::135])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B60DA634F9\n\tfor ;\n\tThu, 26 Sep 2024 18:33:03 +0200 (CEST)","by mail-lf1-x135.google.com with SMTP id\n\t2adb3069b0e04-53690eb134bso123e87.0\n\tfor ;\n\tThu, 26 Sep 2024 09:33:03 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=google.com header.i=@google.com\n\theader.b=\"Ics/6Kd2\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=google.com; s=20230601; t=1727368383; x=1727973183;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=Kl/O621vYauu3f5kvswOOWahkvEKi/TFrWu6o7W2XU0=;\n\tb=Ics/6Kd2+9tK2ZlM/CC/r7qQ/5T2yJnKy4elnhQ2pRcAMD+U6nMK5NbuNpwblEO2ON\n\tMT7TiyrWudSeJ0H+7fvIXaulJHZxQvEKyTyNfLBeQ9lGkFJnZia7FEKFAfXmX2LO/r4S\n\tN4hqFcwNrqIMnXqkFWyeb0+j2cLir5n+3nbOtN2PXQmBaK2/RVzsR1Qbf77IAmL4Xjus\n\tjBq5filqKoofFSqOqSSAvXTil+nPjj0xg1BNZ3DIFtyQBKb3OdExGJohlPfJomo++1FH\n\tYEpOX43HvKQl7VzXbFTR2qDkczHEJVtJlrsDZ410qCpfDUcy5id+IERP5gBYOKHmgknx\n\teZmQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1727368383; x=1727973183;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=Kl/O621vYauu3f5kvswOOWahkvEKi/TFrWu6o7W2XU0=;\n\tb=mLiipa6AnEHTq326/7nszwR82ZiIPF7du3N6zcZu+MPMfKwjQnWcGrTIpkc+8q3tzH\n\ti/QH+WBEVKTst7hDS12u+oD7c7ukBZFLdjhPPTDJpTTiiwxGgclxWx7zkgr1V5Bq01pq\n\tSehRJizghREVd/eGB6tdFx7+bw1o+eSLUT3pld0UQpCLg2ulSqQBOW3aKteyKVf5NvTz\n\tTFl8yGvpOvFZr+ClsZ6eDjsKZDY7AMo+JggaHt0OphD9pSj65Wr42d+nyy77qp+3aJOI\n\trQ1h++J9U1oZUCxzOY+EBXqOnUXTqoM6lwXdUB+ODYXZNxceI51QbtAhdvrEGa+kR3L/\n\tBN7g==","X-Forwarded-Encrypted":"i=1;\n\tAJvYcCXFd5CBXp67QrE4/uTUvUnUvmfag0g4IHU00/hGMnve2GWfElKlXaAnsaHzdxkLHPBLMWJohK4ARjgDFED2b40=@lists.libcamera.org","X-Gm-Message-State":"AOJu0Yzi/zCQByAtYuZxV5A/nClE5qahI3p364xCB3fmet6+YUGTdP07\n\tNeLxAGPcyi+zRBMAF65jJU5YZdmggInyWD4/2xlIFOvMAR8GmTATuEYbOPyRlYmozvdDhyBldiw\n\tTjdE28M+9aZJ8fj+Vx9/qgs5QmlnLabUL+t98","X-Google-Smtp-Source":"AGHT+IFRamM3KxCh4SLH2Nod2pj6eLkxEsG9e/DkghO4Zrb20tg5iGy93ptTPBOVAe71WtM6n+L8XEF2Lhtn1Xu+Crg=","X-Received":"by 2002:a05:6512:ea6:b0:533:49ab:780e with SMTP id\n\t2adb3069b0e04-53897279d62mr308112e87.2.1727368382730; Thu, 26 Sep 2024\n\t09:33:02 -0700 (PDT)","MIME-Version":"1.0","References":"<20240910044834.2477701-1-chenghaoyang@google.com>\n\t<20240910044834.2477701-6-chenghaoyang@google.com>\n\t\n\t<47itxygvnce27mxaqmlyfe7kyzieohzdyu7vztquatmjttklur@66rawijmlq5d>","In-Reply-To":"<47itxygvnce27mxaqmlyfe7kyzieohzdyu7vztquatmjttklur@66rawijmlq5d>","From":"Cheng-Hao Yang ","Date":"Fri, 27 Sep 2024 00:32:26 +0800","Message-ID":"","Subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","To":"Jacopo Mondi ","Cc":"Harvey Yang ,\n\tlibcamera-devel@lists.libcamera.org, \n\tKonami Shu , Yunke Cao , \n\tTomasz Figa ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":31418,"web_url":"https://patchwork.libcamera.org/comment/31418/","msgid":"","date":"2024-09-26T16:32:27","subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":148,"url":"https://patchwork.libcamera.org/api/people/148/","name":"Cheng-Hao Yang","email":"chenghaoyang@google.com"},"content":"Hi Jacopo,\n\nOn Thu, Sep 26, 2024 at 7:31 PM Jacopo Mondi\n wrote:\n>\n> Hi Harvey\n>\n> On Tue, Sep 10, 2024 at 04:40:18AM GMT, Harvey Yang wrote:\n> > Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> > loads real images (jpg / jpeg for now) as the source and generates\n> > scaled frames.\n> >\n> > Signed-off-by: Konami Shu \n> > Co-developed-by: Harvey Yang \n> > Co-developed-by: Yunke Cao \n> > Co-developed-by: Tomasz Figa \n> > ---\n> > .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n> > .../pipeline/virtual/image_frame_generator.h | 51 +++++\n> > src/libcamera/pipeline/virtual/meson.build | 4 +\n> > 3 files changed, 230 insertions(+)\n> > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n> >\n> > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > new file mode 100644\n> > index 00000000..80df9d6b\n> > --- /dev/null\n> > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> > @@ -0,0 +1,175 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2024, Google Inc.\n> > + *\n> > + * image_frame_generator.cpp - Derived class of FrameGenerator for\n> > + * generating frames from images\n> > + */\n> > +\n> > +#include \"image_frame_generator.h\"\n> > +\n> > +#include \n> > +#include \n>\n> Both already included in the header file\n\nRemoved.\n\n>\n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +\n> > +#include \"libcamera/internal/mapped_framebuffer.h\"\n> > +\n> > +#include \"libyuv/convert.h\"\n> > +#include \"libyuv/scale.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +LOG_DECLARE_CATEGORY(Virtual)\n> > +\n> > +/*\n> > + * Factory function to create an ImageFrameGenerator object.\n> > + * Read the images and convert them to buffers in NV12 format.\n> > + * Store the pointers to the buffers to a list (imageFrameDatas)\n> > + */\n> > +std::unique_ptr\n> > +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> > +{\n> > + std::unique_ptr imageFrameGenerator =\n> > + std::make_unique();\n> > + imageFrameGenerator->imageFrames_ = &imageFrames;\n> > +\n> > + /*\n> > + * For each file in the directory, load the image,\n> > + * convert it to NV12, and store the pointer.\n> > + */\n> > + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> > + std::filesystem::path path;\n> > + if (!imageFrames.number)\n> > + /* If the path is to an image */\n> > + path = imageFrames.path;\n> > + else\n> > + /* If the path is to a directory */\n> > + path = imageFrames.path / (std::to_string(i) + \".jpg\");\n> > +\n> > + File file(path);\n> > + if (!file.open(File::OpenModeFlag::ReadOnly)) {\n> > + LOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> > + << \": \" << strerror(file.error());\n> > + return nullptr;\n> > + }\n> > +\n> > + /* Read the image file to data */\n> > + auto fileSize = file.size();\n> > + auto buffer = std::make_unique(file.size());\n> > + if (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> > + LOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> > + << \": \" << strerror(file.error());\n> > + return nullptr;\n> > + }\n> > +\n> > + /* Get the width and height of the image */\n> > + int width, height;\n> > + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> > + LOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> > + << file.fileName();\n> > + return nullptr;\n> > + }\n> > +\n> > + std::unique_ptr dstY =\n> > + std::make_unique(width * height);\n> > + std::unique_ptr dstUV =\n> > + std::make_unique(width * height / 2);\n> > + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> > + dstY.get(), width, dstUV.get(),\n> > + width, width, height, width, height);\n> > + if (ret != 0)\n> > + LOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> > +\n> > + imageFrameGenerator->imageFrameDatas_.emplace_back(\n> > + ImageFrameData{ std::move(dstY), std::move(dstUV),\n> > + Size(width, height) });\n> > + }\n> > +\n> > + return imageFrameGenerator;\n> > +}\n> > +\n> > +/* Scale the buffers for image frames. */\n> > +void ImageFrameGenerator::configure(const Size &size)\n> > +{\n> > + /* Reset the source images to prevent multiple configuration calls */\n> > + scaledFrameDatas_.clear();\n> > + frameCount_ = 0;\n> > + parameter_ = 0;\n> > +\n> > + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> > + /* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> > + unsigned int halfSizeWidth = (size.width + 1) / 2;\n> > + unsigned int halfSizeHeight = (size.height + 1) / 2;\n> > + std::unique_ptr scaledY =\n> > + std::make_unique(size.width * size.height);\n> > + std::unique_ptr scaledUV =\n> > + std::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> > + auto &src = imageFrameDatas_[i];\n> > +\n> > + /*\n> > + * \\todo Some platforms might enforce stride due to GPU, like\n> > + * ChromeOS ciri (64). The weight needs to be a multiple of\n> > + * the stride to work properly for now.\n> > + */\n> > + libyuv::NV12Scale(src.Y.get(), src.size.width,\n> > + src.UV.get(), src.size.width,\n> > + src.size.width, src.size.height,\n> > + scaledY.get(), size.width, scaledUV.get(), size.width,\n> > + size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> > +\n> > + scaledFrameDatas_.emplace_back(\n> > + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> > + }\n> > +}\n> > +\n> > +void ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> > +{\n> > + /* Don't do anything when the list of buffers is empty*/\n> > + ASSERT(!scaledFrameDatas_.empty());\n> > +\n> > + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> > +\n> > + auto planes = mappedFrameBuffer.planes();\n> > +\n> > + /* Make sure the frameCount does not over the number of images */\n> > + frameCount_ %= imageFrames_->number.value_or(1);\n> > +\n> > + /* Write the scaledY and scaledUV to the mapped frame buffer */\n> > + libyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n> > + scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n> > + size.width, planes[1].begin(), size.width,\n> > + size.width, size.height);\n> > +\n> > + /* Proceed an image every 4 frames */\n> > + /* \\todo Consider setting the proceed interval in the config file */\n> > + parameter_++;\n> > + if (parameter_ % frameInterval == 0)\n> > + frameCount_++;\n> > +}\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::imageFrameDatas_\n> > + * \\brief List of pointers to the not scaled image buffers\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::scaledFrameDatas_\n> > + * \\brief List of pointers to the scaled image buffers\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::imageFrames_\n> > + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> > + */\n> > +\n> > +/*\n> > + * \\var ImageFrameGenerator::parameter_\n> > + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> > + */\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > new file mode 100644\n> > index 00000000..bbbe77c8\n> > --- /dev/null\n> > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> > @@ -0,0 +1,51 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2024, Google Inc.\n> > + *\n> > + * image_frame_generator.h - Derived class of FrameGenerator for\n> > + * generating frames from images\n> > + */\n> > +\n> > +#pragma once\n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \"frame_generator.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +/* Frame configuration provided by the config file */\n> > +struct ImageFrames {\n> > + std::filesystem::path path;\n> > + std::optional number;\n> > +};\n> > +\n> > +class ImageFrameGenerator : public FrameGenerator\n> > +{\n> > +public:\n> > + static std::unique_ptr create(ImageFrames &imageFrames);\n> > +\n> > +private:\n> > + static constexpr unsigned int frameInterval = 4;\n>\n> maybe kFrameInterval\n>\n> > +\n> > + struct ImageFrameData {\n> > + std::unique_ptr Y;\n> > + std::unique_ptr UV;\n> > + Size size;\n> > + };\n> > +\n> > + void configure(const Size &size) override;\n> > + void generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> > +\n> > + std::vector imageFrameDatas_;\n> > + std::vector scaledFrameDatas_;\n> > + ImageFrames *imageFrames_;\n> > + unsigned int frameCount_;\n> > + unsigned int parameter_;\n> > +};\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> > index 0c537777..73d10cc3 100644\n> > --- a/src/libcamera/pipeline/virtual/meson.build\n> > +++ b/src/libcamera/pipeline/virtual/meson.build\n> > @@ -1,8 +1,12 @@\n> > # SPDX-License-Identifier: CC0-1.0\n> >\n> > libcamera_internal_sources += files([\n> > + 'image_frame_generator.cpp',\n> > 'test_pattern_generator.cpp',\n> > 'virtual.cpp',\n> > ])\n> >\n> > +libjpeg = dependency('libjpeg', required : false)\n> > +\n>\n> Isn't this required now ?\n\nAh right. Updated.\n\n\n>\n> The rest looks good, thanks!\n>\n> with the above nits fixed\n> Reviewed-by: Jacopo Mondi \n>\n> Thanks\n> j\n>\n>\n> > libcamera_deps += [libyuv_dep]\n> > +libcamera_deps += [libjpeg]\n> > --\n> > 2.46.0.598.g6f2099f65c-goog\n> >\n\n\n\n--\nBR,\nHarvey Yang","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 56CBCC32D4\n\tfor ;\n\tThu, 26 Sep 2024 16:33:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id CCE6963526;\n\tThu, 26 Sep 2024 18:33:12 +0200 (CEST)","from mail-wm1-x32b.google.com (mail-wm1-x32b.google.com\n\t[IPv6:2a00:1450:4864:20::32b])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 6210D63515\n\tfor ;\n\tThu, 26 Sep 2024 18:33:09 +0200 (CEST)","by mail-wm1-x32b.google.com with SMTP id\n\t5b1f17b1804b1-42cb1dd2886so1775e9.0\n\tfor ;\n\tThu, 26 Sep 2024 09:33:09 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=google.com header.i=@google.com\n\theader.b=\"UAhNfaTY\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=google.com; s=20230601; t=1727368389; x=1727973189;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=DXSSvS7e2pz3kS8NeJvmCgTAkjRugbF+scSoZLUOcpA=;\n\tb=UAhNfaTYvZKMaST00VSaSUQBOntpU3w5sI2KCYTCrmZKFIShRj9yLzoS5j76qaZ5Ma\n\tMKFOl7sUL2ZkdhqKTpnkjJ2edtB0okCTSdubmEzdL9pL9M9BWUkeWQW8mkqSQtY3+8Mb\n\tDfI+EBnzK9uHVGhUd4HPTikAnHN7NT9OsN7uy784YIFa6SoKPhdvO8VkmkvTVsJKQWaU\n\tWjW2evUBcpibQ378zpDXrtO7+sg+tvYivLQrV7v9SFBjyTNt6b5QiLoVVd0VZrNvnpT/\n\t0ZYXtbvisznH71tirNv+/01IGKtM9hbi74WCy8q5tyjHhSZic0RT12e1tZMrxBtFwIK0\n\t6HIA==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1727368389; x=1727973189;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=DXSSvS7e2pz3kS8NeJvmCgTAkjRugbF+scSoZLUOcpA=;\n\tb=ZILvkKSHZIOkSxgT+XSOHirDZIw7TF4NrLA6tNOGNrd/m6dSdw12XkCerbGTe3wSnC\n\t9R8pY+L0x2kytrlp7I8B26DnCgqu+84XziVCV3Mdi0zSyr/ozqA8iA1dWfSXqSDwx0K9\n\tGtSZ6EQr8l0S/jVwK2tS/oNFRhzvVH+7mZHAU2HKFGa7BVIeQpeDl5UEvCuxlE3TXJlB\n\tH3hKWSs+vfYBReTxVq9sY+w+vb86akM90++MRts0PHGHbpvMZBlVe+y/aRtxEDml+yz5\n\tf4FlC8bZmtfqghbVAA23tRWcwpK5k5Ux+BeFcdrU3SSmICtpIbfugi5UuZlTSgp6zvl1\n\tU8Sg==","X-Forwarded-Encrypted":"i=1;\n\tAJvYcCUVJuVPYxe+4BfT2vkhjs3/8cQErwzjEugacXYODpU9wi7BpjdLt4ABl2bLFhaBEJFKwqk4W3bH9LXBlG72x6A=@lists.libcamera.org","X-Gm-Message-State":"AOJu0Yyt/QmefGFDkbl5qACfDV7Igfj3QHYO+PvgQsUW2Ze5kPNJJ+wv\n\tkP2aKyHKwSmFIMA0atqROneAyUBOfYUJADD+M5gqSwMaMQ59Ch43Aw/e36WMiBzAv6R8ExPtmMP\n\tMABirN6MKEmXctvpsfm9BfUjdgx5v9bm1NKoJ","X-Google-Smtp-Source":"AGHT+IFEvnYsocwiLz31M6N1DZbvILLd7mfeqGaZRIa4YWiYKoNggFYBlmMJQpgkRqkPVYypmJXUFewW1IbNQGqmx1k=","X-Received":"by 2002:a05:600c:500c:b0:42c:b0b0:513a with SMTP id\n\t5b1f17b1804b1-42f5372db31mr4483675e9.2.1727368385222; Thu, 26 Sep 2024\n\t09:33:05 -0700 (PDT)","MIME-Version":"1.0","References":"<20240910044834.2477701-1-chenghaoyang@google.com>\n\t<20240910044834.2477701-6-chenghaoyang@google.com>\n\t<2m37vv3nlor2ajoa6snibsv3i5k7gjnmla5caxfpoammxxuwaj@awmtffif3mal>","In-Reply-To":"<2m37vv3nlor2ajoa6snibsv3i5k7gjnmla5caxfpoammxxuwaj@awmtffif3mal>","From":"Cheng-Hao Yang ","Date":"Fri, 27 Sep 2024 00:32:27 +0800","Message-ID":"","Subject":"Re: [PATCH v12 5/7] libcamera: virtual: Add ImageFrameGenerator","To":"Jacopo Mondi ","Cc":"Harvey Yang ,\n\tlibcamera-devel@lists.libcamera.org, \n\tKonami Shu , Yunke Cao , \n\tTomasz Figa ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]