[{"id":31891,"web_url":"https://patchwork.libcamera.org/comment/31891/","msgid":"<172969202857.2597140.14587597764256691270@ping.linuxembedded.co.uk>","date":"2024-10-23T14:00:28","subject":"Re: [PATCH v16 5/7] libcamera: virtual: Add ImageFrameGenerator","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Harvey Yang (2024-10-22 08:43:41)\n> Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\n> loads real images (jpg / jpeg for now) as the source and generates\n> scaled frames.\n> \n> Signed-off-by: Konami Shu \n> Co-developed-by: Harvey Yang \n> Signed-off-by: Harvey Yang \n> Co-developed-by: Yunke Cao \n> Signed-off-by: Yunke Cao \n> Co-developed-by: Tomasz Figa \n> Signed-off-by: Tomasz Figa \n> Reviewed-by: Jacopo Mondi \n\n\nReviewed-by: Kieran Bingham \n\n> ---\n> .../virtual/image_frame_generator.cpp | 178 ++++++++++++++++++\n> .../pipeline/virtual/image_frame_generator.h | 50 +++++\n> src/libcamera/pipeline/virtual/meson.build | 4 +\n> 3 files changed, 232 insertions(+)\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h\n> \n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> new file mode 100644\n> index 000000000..e140969c8\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n> @@ -0,0 +1,178 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * Derived class of FrameGenerator for generating frames from images\n> + */\n> +\n> +#include \"image_frame_generator.h\"\n> +\n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#include \"libcamera/internal/mapped_framebuffer.h\"\n> +\n> +#include \"libyuv/convert.h\"\n> +#include \"libyuv/scale.h\"\n> +\n> +namespace libcamera {\n> +\n> +LOG_DECLARE_CATEGORY(Virtual)\n> +\n> +/*\n> + * Factory function to create an ImageFrameGenerator object.\n> + * Read the images and convert them to buffers in NV12 format.\n> + * Store the pointers to the buffers to a list (imageFrameDatas)\n> + */\n> +std::unique_ptr\n> +ImageFrameGenerator::create(ImageFrames &imageFrames)\n> +{\n> + std::unique_ptr imageFrameGenerator =\n> + std::make_unique();\n> + imageFrameGenerator->imageFrames_ = &imageFrames;\n> +\n> + /*\n> + * For each file in the directory, load the image,\n> + * convert it to NV12, and store the pointer.\n> + */\n> + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n> + std::filesystem::path path;\n> + if (!imageFrames.number)\n> + /* If the path is to an image */\n> + path = imageFrames.path;\n> + else\n> + /* If the path is to a directory */\n> + path = imageFrames.path / (std::to_string(i) + \".jpg\");\n> +\n> + File file(path);\n> + if (!file.open(File::OpenModeFlag::ReadOnly)) {\n> + LOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n> + << \": \" << strerror(file.error());\n> + return nullptr;\n> + }\n> +\n> + /* Read the image file to data */\n> + auto fileSize = file.size();\n> + auto buffer = std::make_unique(fileSize);\n> + if (file.read({ buffer.get(), static_cast(fileSize) }) != fileSize) {\n> + LOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n> + << \": \" << strerror(file.error());\n> + return nullptr;\n> + }\n> +\n> + /* Get the width and height of the image */\n> + int width, height;\n> + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n> + LOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n> + << file.fileName();\n> + return nullptr;\n> + }\n> +\n> + std::unique_ptr dstY =\n> + std::make_unique(width * height);\n> + std::unique_ptr dstUV =\n> + std::make_unique(width * height / 2);\n> + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n> + dstY.get(), width, dstUV.get(),\n> + width, width, height, width, height);\n> + if (ret != 0)\n> + LOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n> +\n> + imageFrameGenerator->imageFrameDatas_.emplace_back(\n> + ImageFrameData{ std::move(dstY), std::move(dstUV),\n> + Size(width, height) });\n> + }\n> +\n> + return imageFrameGenerator;\n> +}\n> +\n> +/*\n> + * \\var ImageFrameGenerator::frameRepeat\n> + * \\brief Number of frames to repeat before proceeding to the next frame\n> + */\n> +\n> +/* Scale the buffers for image frames. */\n> +void ImageFrameGenerator::configure(const Size &size)\n> +{\n> + /* Reset the source images to prevent multiple configuration calls */\n> + scaledFrameDatas_.clear();\n> + frameIndex_ = 0;\n> + parameter_ = 0;\n> +\n> + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n> + /* Scale the imageFrameDatas_ to scaledY and scaledUV */\n> + unsigned int halfSizeWidth = (size.width + 1) / 2;\n> + unsigned int halfSizeHeight = (size.height + 1) / 2;\n> + std::unique_ptr scaledY =\n> + std::make_unique(size.width * size.height);\n> + std::unique_ptr scaledUV =\n> + std::make_unique(halfSizeWidth * halfSizeHeight * 2);\n> + auto &src = imageFrameDatas_[i];\n> +\n> + /*\n> + * \\todo Some platforms might enforce stride due to GPU.\n> + * The width needs to be a multiple of the stride to work\n> + * properly for now.\n> + */\n> + libyuv::NV12Scale(src.Y.get(), src.size.width,\n> + src.UV.get(), src.size.width,\n> + src.size.width, src.size.height,\n> + scaledY.get(), size.width, scaledUV.get(), size.width,\n> + size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n> +\n> + scaledFrameDatas_.emplace_back(\n> + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n> + }\n> +}\n> +\n> +int ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n> +{\n> + ASSERT(!scaledFrameDatas_.empty());\n> +\n> + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n> +\n> + auto planes = mappedFrameBuffer.planes();\n> +\n> + /* Loop only around the number of images available */\n> + frameIndex_ %= imageFrames_->number.value_or(1);\n> +\n> + /* Write the scaledY and scaledUV to the mapped frame buffer */\n> + libyuv::NV12Copy(scaledFrameDatas_[frameIndex_].Y.get(), size.width,\n> + scaledFrameDatas_[frameIndex_].UV.get(), size.width, planes[0].begin(),\n> + size.width, planes[1].begin(), size.width,\n> + size.width, size.height);\n> +\n> + /* Proceed to the next image every 4 frames */\n> + /* \\todo Consider setting the frameRepeat in the config file */\n> + parameter_++;\n> + if (parameter_ % frameRepeat == 0)\n> + frameIndex_++;\n> +\n> + return 0;\n> +}\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrameDatas_\n> + * \\brief List of pointers to the not scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::scaledFrameDatas_\n> + * \\brief List of pointers to the scaled image buffers\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::imageFrames_\n> + * \\brief Pointer to the imageFrames_ in VirtualCameraData\n> + */\n> +\n> +/*\n> + * \\var ImageFrameGenerator::parameter_\n> + * \\brief Speed parameter. Change to the next image every parameter_ frames\n> + */\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> new file mode 100644\n> index 000000000..e072a47b8\n> --- /dev/null\n> +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n> @@ -0,0 +1,50 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Google Inc.\n> + *\n> + * Derived class of FrameGenerator for generating frames from images\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +\n> +#include \"frame_generator.h\"\n> +\n> +namespace libcamera {\n> +\n> +/* Frame configuration provided by the config file */\n> +struct ImageFrames {\n> + std::filesystem::path path;\n> + std::optional number;\n> +};\n> +\n> +class ImageFrameGenerator : public FrameGenerator\n> +{\n> +public:\n> + static std::unique_ptr create(ImageFrames &imageFrames);\n> +\n> +private:\n> + static constexpr unsigned int frameRepeat = 4;\n> +\n> + struct ImageFrameData {\n> + std::unique_ptr Y;\n> + std::unique_ptr UV;\n> + Size size;\n> + };\n> +\n> + void configure(const Size &size) override;\n> + int generateFrame(const Size &size, const FrameBuffer *buffer) override;\n> +\n> + std::vector imageFrameDatas_;\n> + std::vector scaledFrameDatas_;\n> + ImageFrames *imageFrames_;\n> + unsigned int frameIndex_;\n> + unsigned int parameter_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\n> index 0c537777f..bb38c193c 100644\n> --- a/src/libcamera/pipeline/virtual/meson.build\n> +++ b/src/libcamera/pipeline/virtual/meson.build\n> @@ -1,8 +1,12 @@\n> # SPDX-License-Identifier: CC0-1.0\n> \n> libcamera_internal_sources += files([\n> + 'image_frame_generator.cpp',\n> 'test_pattern_generator.cpp',\n> 'virtual.cpp',\n> ])\n> \n> +libjpeg = dependency('libjpeg', required : true)\n> +\n> libcamera_deps += [libyuv_dep]\n> +libcamera_deps += [libjpeg]\n> -- \n> 2.47.0.105.g07ac214952-goog\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D0EABC3213\n\tfor ;\n\tWed, 23 Oct 2024 14:00:35 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id BE43165393;\n\tWed, 23 Oct 2024 16:00:34 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2D4C86537E\n\tfor ;\n\tWed, 23 Oct 2024 16:00:33 +0200 (CEST)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id D26EFA47;\n\tWed, 23 Oct 2024 15:58:43 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"nZjWKSFF\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1729691923;\n\tbh=9eSzdIXwgYLaWval0+Dcfz7dFjQ8U3mMOhkUMrWSoq8=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=nZjWKSFFQ0Jzyx8BIn2zuYxmR/LseBd1Qeyc0YYvTWOw/3g6cfCdIfTEbRhScosCN\n\tin8gSbdP8Yx7xUIDnGKoQmSygTec2LJ7TKZGtZHfH78m4u4uXKAQsvk4cnY46WDtED\n\tQVmWafixI+cr+Fh6JaihE7xGCt5qCifiEkfJ404U=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20241022074544.3790451-6-chenghaoyang@chromium.org>","References":"<20241022074544.3790451-1-chenghaoyang@chromium.org>\n\t<20241022074544.3790451-6-chenghaoyang@chromium.org>","Subject":"Re: [PATCH v16 5/7] libcamera: virtual: Add ImageFrameGenerator","From":"Kieran Bingham ","Cc":"Harvey Yang , Konami Shu ,\n\tYunke Cao , Tomasz Figa ,\n\tJacopo Mondi ","To":"Harvey Yang ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Wed, 23 Oct 2024 15:00:28 +0100","Message-ID":"<172969202857.2597140.14587597764256691270@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]