| Message ID | 20241022074544.3790451-6-chenghaoyang@chromium.org |
|---|---|
| State | Accepted |
| Headers | show |
| Series |
|
| Related | show |
Quoting Harvey Yang (2024-10-22 08:43:41) > Besides TestPatternGenerator, this patch adds ImageFrameGenerator that > loads real images (jpg / jpeg for now) as the source and generates > scaled frames. > > Signed-off-by: Konami Shu <konamiz@google.com> > Co-developed-by: Harvey Yang <chenghaoyang@chromium.org> > Signed-off-by: Harvey Yang <chenghaoyang@chromium.org> > Co-developed-by: Yunke Cao <yunkec@chromium.org> > Signed-off-by: Yunke Cao <yunkec@chromium.org> > Co-developed-by: Tomasz Figa <tfiga@chromium.org> > Signed-off-by: Tomasz Figa <tfiga@chromium.org> > Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> > --- > .../virtual/image_frame_generator.cpp | 178 ++++++++++++++++++ > .../pipeline/virtual/image_frame_generator.h | 50 +++++ > src/libcamera/pipeline/virtual/meson.build | 4 + > 3 files changed, 232 insertions(+) > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp > create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h > > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp > new file mode 100644 > index 000000000..e140969c8 > --- /dev/null > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp > @@ -0,0 +1,178 @@ > +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > +/* > + * Copyright (C) 2024, Google Inc. > + * > + * Derived class of FrameGenerator for generating frames from images > + */ > + > +#include "image_frame_generator.h" > + > +#include <string> > + > +#include <libcamera/base/file.h> > +#include <libcamera/base/log.h> > + > +#include <libcamera/framebuffer.h> > + > +#include "libcamera/internal/mapped_framebuffer.h" > + > +#include "libyuv/convert.h" > +#include "libyuv/scale.h" > + > +namespace libcamera { > + > +LOG_DECLARE_CATEGORY(Virtual) > + > +/* > + * Factory function to create an ImageFrameGenerator object. > + * Read the images and convert them to buffers in NV12 format. > + * Store the pointers to the buffers to a list (imageFrameDatas) > + */ > +std::unique_ptr<ImageFrameGenerator> > +ImageFrameGenerator::create(ImageFrames &imageFrames) > +{ > + std::unique_ptr<ImageFrameGenerator> imageFrameGenerator = > + std::make_unique<ImageFrameGenerator>(); > + imageFrameGenerator->imageFrames_ = &imageFrames; > + > + /* > + * For each file in the directory, load the image, > + * convert it to NV12, and store the pointer. > + */ > + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) { > + std::filesystem::path path; > + if (!imageFrames.number) > + /* If the path is to an image */ > + path = imageFrames.path; > + else > + /* If the path is to a directory */ > + path = imageFrames.path / (std::to_string(i) + ".jpg"); > + > + File file(path); > + if (!file.open(File::OpenModeFlag::ReadOnly)) { > + LOG(Virtual, Error) << "Failed to open image file " << file.fileName() > + << ": " << strerror(file.error()); > + return nullptr; > + } > + > + /* Read the image file to data */ > + auto fileSize = file.size(); > + auto buffer = std::make_unique<uint8_t[]>(fileSize); > + if (file.read({ buffer.get(), static_cast<size_t>(fileSize) }) != fileSize) { > + LOG(Virtual, Error) << "Failed to read file " << file.fileName() > + << ": " << strerror(file.error()); > + return nullptr; > + } > + > + /* Get the width and height of the image */ > + int width, height; > + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) { > + LOG(Virtual, Error) << "Failed to get the size of the image file: " > + << file.fileName(); > + return nullptr; > + } > + > + std::unique_ptr<uint8_t[]> dstY = > + std::make_unique<uint8_t[]>(width * height); > + std::unique_ptr<uint8_t[]> dstUV = > + std::make_unique<uint8_t[]>(width * height / 2); > + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize, > + dstY.get(), width, dstUV.get(), > + width, width, height, width, height); > + if (ret != 0) > + LOG(Virtual, Error) << "MJPGToNV12() failed with " << ret; > + > + imageFrameGenerator->imageFrameDatas_.emplace_back( > + ImageFrameData{ std::move(dstY), std::move(dstUV), > + Size(width, height) }); > + } > + > + return imageFrameGenerator; > +} > + > +/* > + * \var ImageFrameGenerator::frameRepeat > + * \brief Number of frames to repeat before proceeding to the next frame > + */ > + > +/* Scale the buffers for image frames. */ > +void ImageFrameGenerator::configure(const Size &size) > +{ > + /* Reset the source images to prevent multiple configuration calls */ > + scaledFrameDatas_.clear(); > + frameIndex_ = 0; > + parameter_ = 0; > + > + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) { > + /* Scale the imageFrameDatas_ to scaledY and scaledUV */ > + unsigned int halfSizeWidth = (size.width + 1) / 2; > + unsigned int halfSizeHeight = (size.height + 1) / 2; > + std::unique_ptr<uint8_t[]> scaledY = > + std::make_unique<uint8_t[]>(size.width * size.height); > + std::unique_ptr<uint8_t[]> scaledUV = > + std::make_unique<uint8_t[]>(halfSizeWidth * halfSizeHeight * 2); > + auto &src = imageFrameDatas_[i]; > + > + /* > + * \todo Some platforms might enforce stride due to GPU. > + * The width needs to be a multiple of the stride to work > + * properly for now. > + */ > + libyuv::NV12Scale(src.Y.get(), src.size.width, > + src.UV.get(), src.size.width, > + src.size.width, src.size.height, > + scaledY.get(), size.width, scaledUV.get(), size.width, > + size.width, size.height, libyuv::FilterMode::kFilterBilinear); > + > + scaledFrameDatas_.emplace_back( > + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size }); > + } > +} > + > +int ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer) > +{ > + ASSERT(!scaledFrameDatas_.empty()); > + > + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write); > + > + auto planes = mappedFrameBuffer.planes(); > + > + /* Loop only around the number of images available */ > + frameIndex_ %= imageFrames_->number.value_or(1); > + > + /* Write the scaledY and scaledUV to the mapped frame buffer */ > + libyuv::NV12Copy(scaledFrameDatas_[frameIndex_].Y.get(), size.width, > + scaledFrameDatas_[frameIndex_].UV.get(), size.width, planes[0].begin(), > + size.width, planes[1].begin(), size.width, > + size.width, size.height); > + > + /* Proceed to the next image every 4 frames */ > + /* \todo Consider setting the frameRepeat in the config file */ > + parameter_++; > + if (parameter_ % frameRepeat == 0) > + frameIndex_++; > + > + return 0; > +} > + > +/* > + * \var ImageFrameGenerator::imageFrameDatas_ > + * \brief List of pointers to the not scaled image buffers > + */ > + > +/* > + * \var ImageFrameGenerator::scaledFrameDatas_ > + * \brief List of pointers to the scaled image buffers > + */ > + > +/* > + * \var ImageFrameGenerator::imageFrames_ > + * \brief Pointer to the imageFrames_ in VirtualCameraData > + */ > + > +/* > + * \var ImageFrameGenerator::parameter_ > + * \brief Speed parameter. Change to the next image every parameter_ frames > + */ > + > +} /* namespace libcamera */ > diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h > new file mode 100644 > index 000000000..e072a47b8 > --- /dev/null > +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h > @@ -0,0 +1,50 @@ > +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > +/* > + * Copyright (C) 2024, Google Inc. > + * > + * Derived class of FrameGenerator for generating frames from images > + */ > + > +#pragma once > + > +#include <filesystem> > +#include <memory> > +#include <optional> > +#include <stdint.h> > +#include <sys/types.h> > + > +#include "frame_generator.h" > + > +namespace libcamera { > + > +/* Frame configuration provided by the config file */ > +struct ImageFrames { > + std::filesystem::path path; > + std::optional<unsigned int> number; > +}; > + > +class ImageFrameGenerator : public FrameGenerator > +{ > +public: > + static std::unique_ptr<ImageFrameGenerator> create(ImageFrames &imageFrames); > + > +private: > + static constexpr unsigned int frameRepeat = 4; > + > + struct ImageFrameData { > + std::unique_ptr<uint8_t[]> Y; > + std::unique_ptr<uint8_t[]> UV; > + Size size; > + }; > + > + void configure(const Size &size) override; > + int generateFrame(const Size &size, const FrameBuffer *buffer) override; > + > + std::vector<ImageFrameData> imageFrameDatas_; > + std::vector<ImageFrameData> scaledFrameDatas_; > + ImageFrames *imageFrames_; > + unsigned int frameIndex_; > + unsigned int parameter_; > +}; > + > +} /* namespace libcamera */ > diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build > index 0c537777f..bb38c193c 100644 > --- a/src/libcamera/pipeline/virtual/meson.build > +++ b/src/libcamera/pipeline/virtual/meson.build > @@ -1,8 +1,12 @@ > # SPDX-License-Identifier: CC0-1.0 > > libcamera_internal_sources += files([ > + 'image_frame_generator.cpp', > 'test_pattern_generator.cpp', > 'virtual.cpp', > ]) > > +libjpeg = dependency('libjpeg', required : true) > + > libcamera_deps += [libyuv_dep] > +libcamera_deps += [libjpeg] > -- > 2.47.0.105.g07ac214952-goog >
diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp new file mode 100644 index 000000000..e140969c8 --- /dev/null +++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Google Inc. + * + * Derived class of FrameGenerator for generating frames from images + */ + +#include "image_frame_generator.h" + +#include <string> + +#include <libcamera/base/file.h> +#include <libcamera/base/log.h> + +#include <libcamera/framebuffer.h> + +#include "libcamera/internal/mapped_framebuffer.h" + +#include "libyuv/convert.h" +#include "libyuv/scale.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(Virtual) + +/* + * Factory function to create an ImageFrameGenerator object. + * Read the images and convert them to buffers in NV12 format. + * Store the pointers to the buffers to a list (imageFrameDatas) + */ +std::unique_ptr<ImageFrameGenerator> +ImageFrameGenerator::create(ImageFrames &imageFrames) +{ + std::unique_ptr<ImageFrameGenerator> imageFrameGenerator = + std::make_unique<ImageFrameGenerator>(); + imageFrameGenerator->imageFrames_ = &imageFrames; + + /* + * For each file in the directory, load the image, + * convert it to NV12, and store the pointer. + */ + for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) { + std::filesystem::path path; + if (!imageFrames.number) + /* If the path is to an image */ + path = imageFrames.path; + else + /* If the path is to a directory */ + path = imageFrames.path / (std::to_string(i) + ".jpg"); + + File file(path); + if (!file.open(File::OpenModeFlag::ReadOnly)) { + LOG(Virtual, Error) << "Failed to open image file " << file.fileName() + << ": " << strerror(file.error()); + return nullptr; + } + + /* Read the image file to data */ + auto fileSize = file.size(); + auto buffer = std::make_unique<uint8_t[]>(fileSize); + if (file.read({ buffer.get(), static_cast<size_t>(fileSize) }) != fileSize) { + LOG(Virtual, Error) << "Failed to read file " << file.fileName() + << ": " << strerror(file.error()); + return nullptr; + } + + /* Get the width and height of the image */ + int width, height; + if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) { + LOG(Virtual, Error) << "Failed to get the size of the image file: " + << file.fileName(); + return nullptr; + } + + std::unique_ptr<uint8_t[]> dstY = + std::make_unique<uint8_t[]>(width * height); + std::unique_ptr<uint8_t[]> dstUV = + std::make_unique<uint8_t[]>(width * height / 2); + int ret = libyuv::MJPGToNV12(buffer.get(), fileSize, + dstY.get(), width, dstUV.get(), + width, width, height, width, height); + if (ret != 0) + LOG(Virtual, Error) << "MJPGToNV12() failed with " << ret; + + imageFrameGenerator->imageFrameDatas_.emplace_back( + ImageFrameData{ std::move(dstY), std::move(dstUV), + Size(width, height) }); + } + + return imageFrameGenerator; +} + +/* + * \var ImageFrameGenerator::frameRepeat + * \brief Number of frames to repeat before proceeding to the next frame + */ + +/* Scale the buffers for image frames. */ +void ImageFrameGenerator::configure(const Size &size) +{ + /* Reset the source images to prevent multiple configuration calls */ + scaledFrameDatas_.clear(); + frameIndex_ = 0; + parameter_ = 0; + + for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) { + /* Scale the imageFrameDatas_ to scaledY and scaledUV */ + unsigned int halfSizeWidth = (size.width + 1) / 2; + unsigned int halfSizeHeight = (size.height + 1) / 2; + std::unique_ptr<uint8_t[]> scaledY = + std::make_unique<uint8_t[]>(size.width * size.height); + std::unique_ptr<uint8_t[]> scaledUV = + std::make_unique<uint8_t[]>(halfSizeWidth * halfSizeHeight * 2); + auto &src = imageFrameDatas_[i]; + + /* + * \todo Some platforms might enforce stride due to GPU. + * The width needs to be a multiple of the stride to work + * properly for now. + */ + libyuv::NV12Scale(src.Y.get(), src.size.width, + src.UV.get(), src.size.width, + src.size.width, src.size.height, + scaledY.get(), size.width, scaledUV.get(), size.width, + size.width, size.height, libyuv::FilterMode::kFilterBilinear); + + scaledFrameDatas_.emplace_back( + ImageFrameData{ std::move(scaledY), std::move(scaledUV), size }); + } +} + +int ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer) +{ + ASSERT(!scaledFrameDatas_.empty()); + + MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write); + + auto planes = mappedFrameBuffer.planes(); + + /* Loop only around the number of images available */ + frameIndex_ %= imageFrames_->number.value_or(1); + + /* Write the scaledY and scaledUV to the mapped frame buffer */ + libyuv::NV12Copy(scaledFrameDatas_[frameIndex_].Y.get(), size.width, + scaledFrameDatas_[frameIndex_].UV.get(), size.width, planes[0].begin(), + size.width, planes[1].begin(), size.width, + size.width, size.height); + + /* Proceed to the next image every 4 frames */ + /* \todo Consider setting the frameRepeat in the config file */ + parameter_++; + if (parameter_ % frameRepeat == 0) + frameIndex_++; + + return 0; +} + +/* + * \var ImageFrameGenerator::imageFrameDatas_ + * \brief List of pointers to the not scaled image buffers + */ + +/* + * \var ImageFrameGenerator::scaledFrameDatas_ + * \brief List of pointers to the scaled image buffers + */ + +/* + * \var ImageFrameGenerator::imageFrames_ + * \brief Pointer to the imageFrames_ in VirtualCameraData + */ + +/* + * \var ImageFrameGenerator::parameter_ + * \brief Speed parameter. Change to the next image every parameter_ frames + */ + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h new file mode 100644 index 000000000..e072a47b8 --- /dev/null +++ b/src/libcamera/pipeline/virtual/image_frame_generator.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Google Inc. + * + * Derived class of FrameGenerator for generating frames from images + */ + +#pragma once + +#include <filesystem> +#include <memory> +#include <optional> +#include <stdint.h> +#include <sys/types.h> + +#include "frame_generator.h" + +namespace libcamera { + +/* Frame configuration provided by the config file */ +struct ImageFrames { + std::filesystem::path path; + std::optional<unsigned int> number; +}; + +class ImageFrameGenerator : public FrameGenerator +{ +public: + static std::unique_ptr<ImageFrameGenerator> create(ImageFrames &imageFrames); + +private: + static constexpr unsigned int frameRepeat = 4; + + struct ImageFrameData { + std::unique_ptr<uint8_t[]> Y; + std::unique_ptr<uint8_t[]> UV; + Size size; + }; + + void configure(const Size &size) override; + int generateFrame(const Size &size, const FrameBuffer *buffer) override; + + std::vector<ImageFrameData> imageFrameDatas_; + std::vector<ImageFrameData> scaledFrameDatas_; + ImageFrames *imageFrames_; + unsigned int frameIndex_; + unsigned int parameter_; +}; + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build index 0c537777f..bb38c193c 100644 --- a/src/libcamera/pipeline/virtual/meson.build +++ b/src/libcamera/pipeline/virtual/meson.build @@ -1,8 +1,12 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_sources += files([ + 'image_frame_generator.cpp', 'test_pattern_generator.cpp', 'virtual.cpp', ]) +libjpeg = dependency('libjpeg', required : true) + libcamera_deps += [libyuv_dep] +libcamera_deps += [libjpeg]