Patch Detail
Show a patch.
GET /api/1.1/patches/21419/?format=api
{ "id": 21419, "url": "https://patchwork.libcamera.org/api/1.1/patches/21419/?format=api", "web_url": "https://patchwork.libcamera.org/patch/21419/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20240930063342.3014837-6-chenghaoyang@google.com>", "date": "2024-09-30T06:29:46", "name": "[v14,5/7] libcamera: virtual: Add ImageFrameGenerator", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "2891d4785ef3a77bcb04f7417d387ec71caef2d6", "submitter": { "id": 117, "url": "https://patchwork.libcamera.org/api/1.1/people/117/?format=api", "name": "Cheng-Hao Yang", "email": "chenghaoyang@chromium.org" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/21419/mbox/", "series": [ { "id": 4637, "url": "https://patchwork.libcamera.org/api/1.1/series/4637/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4637", "date": "2024-09-30T06:29:41", "name": "Add VirtualPipelineHandler", "version": 14, "mbox": "https://patchwork.libcamera.org/series/4637/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/21419/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/21419/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 2F46CC3257\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 30 Sep 2024 06:34:13 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 97D6B63522;\n\tMon, 30 Sep 2024 08:34:12 +0200 (CEST)", "from mail-ot1-x32d.google.com (mail-ot1-x32d.google.com\n\t[IPv6:2607:f8b0:4864:20::32d])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BFE5363512\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 30 Sep 2024 08:33:58 +0200 (CEST)", "by mail-ot1-x32d.google.com with SMTP id\n\t46e09a7af769-710daaadd9bso2212774a34.2\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tSun, 29 Sep 2024 23:33:58 -0700 (PDT)", "from chenghaoyang-low.c.googlers.com.com\n\t(208.158.221.35.bc.googleusercontent.com. [35.221.158.208])\n\tby smtp.gmail.com with ESMTPSA id\n\td2e1a72fcca58-71b265166e2sm5487671b3a.131.2024.09.29.23.33.54\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tSun, 29 Sep 2024 23:33:56 -0700 (PDT)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"Rx6kYfnl\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=chromium.org; s=google; t=1727678036; x=1728282836;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=FKROXjaRq7rbZO9SmMn9LKER7xG7vG5xkgDrpVjuMfo=;\n\tb=Rx6kYfnl0HnqwAxf/dQXgaGv+ywEIJBUhreglXjVydhRMkZbnoSujTV+BHVFkdZPPK\n\tTQaDF6jKZADl+cf90oy3EplC0ADgCHDg+zi/wPGRSXAT1eGppUE24YBtEGbkAFt1EpdE\n\tBuvUaIEzZDwxdYI6re5N9o15xG2lNYY+M9Axo=", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1727678036; x=1728282836;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=FKROXjaRq7rbZO9SmMn9LKER7xG7vG5xkgDrpVjuMfo=;\n\tb=mir2OxDLY4Fhp2bMrPp3dp9ykF+/jLPt7+Urp98pcCxeYU6EDLlszvddq1YmXItr4J\n\tillVILZtObGSmQHrWz+0G86EywT4xzUC4Kk0J0SO3+NcUX7SrJxtO1fP4cSjaq0KLxzG\n\ty49v3OPs6uP8MmcJ6Une7/ChhWcAxxL5PNHakDQNLQ9uiaa4JOyH7ixUJeJHwFbi4jU7\n\tdeaKXAuR2Fjcj06b1fWiYVaX05LTdV+Ii0IzUcaYQSw4gthyJ5DHntb5xB/sGuqlII+b\n\tM4YLhsPrmuJQewzLL6MdUBKI/KINY4LUBvq0zGhc7rWwfUOPtdVuwrJz+g7tPt3fyaxM\n\tF4uA==", "X-Gm-Message-State": "AOJu0Yzljt3/d2+Ns/HJGiNR6bAGn8CqnmCKnKdS6tlIbQD6qvxu6YyM\n\tKS7RuCMlkURpJjL3FFT7SRn6Kzv1iuC8X+FPu6ZwN7Rq3a5JpS92Fb4l7agzu0qpIj4I6FGs46E\n\t=", "X-Google-Smtp-Source": "AGHT+IGCQcr2ErVt8xJDNYcYG7VTdSbXIpRBORAvdbLdwRRfFWJ+08o5VGrqJu5xIjeeRRp6NMh6CA==", "X-Received": "by 2002:a05:6830:3592:b0:710:fae8:fcae with SMTP id\n\t46e09a7af769-714fbe908cbmr8497296a34.18.1727678036680; \n\tSun, 29 Sep 2024 23:33:56 -0700 (PDT)", "From": "Harvey Yang <chenghaoyang@chromium.org>", "X-Google-Original-From": "Harvey Yang <chenghaoyang@google.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Harvey Yang <chenghaoyang@google.com>, Konami Shu <konamiz@google.com>, \n\tHarvey Yang <chenghaoyang@chromium.org>, Yunke Cao <yunkec@chromium.org>,\n\tTomasz Figa <tfiga@chromium.org>,\n\tJacopo Mondi <jacopo.mondi@ideasonboard.com>", "Subject": "[PATCH v14 5/7] libcamera: virtual: Add ImageFrameGenerator", "Date": "Mon, 30 Sep 2024 06:29:46 +0000", "Message-ID": "<20240930063342.3014837-6-chenghaoyang@google.com>", "X-Mailer": "git-send-email 2.46.1.824.gd892dcdcdd-goog", "In-Reply-To": "<20240930063342.3014837-1-chenghaoyang@google.com>", "References": "<20240930063342.3014837-1-chenghaoyang@google.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Besides TestPatternGenerator, this patch adds ImageFrameGenerator that\nloads real images (jpg / jpeg for now) as the source and generates\nscaled frames.\n\nSigned-off-by: Konami Shu <konamiz@google.com>\nCo-developed-by: Harvey Yang <chenghaoyang@chromium.org>\nCo-developed-by: Yunke Cao <yunkec@chromium.org>\nCo-developed-by: Tomasz Figa <tfiga@chromium.org>\nReviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\n---\n .../virtual/image_frame_generator.cpp | 175 ++++++++++++++++++\n .../pipeline/virtual/image_frame_generator.h | 51 +++++\n src/libcamera/pipeline/virtual/meson.build | 4 +\n 3 files changed, 230 insertions(+)\n create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp\n create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h", "diff": "diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\nnew file mode 100644\nindex 00000000..a1915b24\n--- /dev/null\n+++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp\n@@ -0,0 +1,175 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Google Inc.\n+ *\n+ * image_frame_generator.cpp - Derived class of FrameGenerator for\n+ * generating frames from images\n+ */\n+\n+#include \"image_frame_generator.h\"\n+\n+#include <string>\n+\n+#include <libcamera/base/file.h>\n+#include <libcamera/base/log.h>\n+\n+#include <libcamera/framebuffer.h>\n+\n+#include \"libcamera/internal/mapped_framebuffer.h\"\n+\n+#include \"libyuv/convert.h\"\n+#include \"libyuv/scale.h\"\n+\n+namespace libcamera {\n+\n+LOG_DECLARE_CATEGORY(Virtual)\n+\n+/*\n+ * Factory function to create an ImageFrameGenerator object.\n+ * Read the images and convert them to buffers in NV12 format.\n+ * Store the pointers to the buffers to a list (imageFrameDatas)\n+ */\n+std::unique_ptr<ImageFrameGenerator>\n+ImageFrameGenerator::create(ImageFrames &imageFrames)\n+{\n+\tstd::unique_ptr<ImageFrameGenerator> imageFrameGenerator =\n+\t\tstd::make_unique<ImageFrameGenerator>();\n+\timageFrameGenerator->imageFrames_ = &imageFrames;\n+\n+\t/*\n+\t * For each file in the directory, load the image,\n+\t * convert it to NV12, and store the pointer.\n+\t */\n+\tfor (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {\n+\t\tstd::filesystem::path path;\n+\t\tif (!imageFrames.number)\n+\t\t\t/* If the path is to an image */\n+\t\t\tpath = imageFrames.path;\n+\t\telse\n+\t\t\t/* If the path is to a directory */\n+\t\t\tpath = imageFrames.path / (std::to_string(i) + \".jpg\");\n+\n+\t\tFile file(path);\n+\t\tif (!file.open(File::OpenModeFlag::ReadOnly)) {\n+\t\t\tLOG(Virtual, Error) << \"Failed to open image file \" << file.fileName()\n+\t\t\t\t\t << \": \" << strerror(file.error());\n+\t\t\treturn nullptr;\n+\t\t}\n+\n+\t\t/* Read the image file to data */\n+\t\tauto fileSize = file.size();\n+\t\tauto buffer = std::make_unique<uint8_t[]>(file.size());\n+\t\tif (file.read({ buffer.get(), static_cast<size_t>(fileSize) }) != fileSize) {\n+\t\t\tLOG(Virtual, Error) << \"Failed to read file \" << file.fileName()\n+\t\t\t\t\t << \": \" << strerror(file.error());\n+\t\t\treturn nullptr;\n+\t\t}\n+\n+\t\t/* Get the width and height of the image */\n+\t\tint width, height;\n+\t\tif (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {\n+\t\t\tLOG(Virtual, Error) << \"Failed to get the size of the image file: \"\n+\t\t\t\t\t << file.fileName();\n+\t\t\treturn nullptr;\n+\t\t}\n+\n+\t\tstd::unique_ptr<uint8_t[]> dstY =\n+\t\t\tstd::make_unique<uint8_t[]>(width * height);\n+\t\tstd::unique_ptr<uint8_t[]> dstUV =\n+\t\t\tstd::make_unique<uint8_t[]>(width * height / 2);\n+\t\tint ret = libyuv::MJPGToNV12(buffer.get(), fileSize,\n+\t\t\t\t\t dstY.get(), width, dstUV.get(),\n+\t\t\t\t\t width, width, height, width, height);\n+\t\tif (ret != 0)\n+\t\t\tLOG(Virtual, Error) << \"MJPGToNV12() failed with \" << ret;\n+\n+\t\timageFrameGenerator->imageFrameDatas_.emplace_back(\n+\t\t\tImageFrameData{ std::move(dstY), std::move(dstUV),\n+\t\t\t\t\tSize(width, height) });\n+\t}\n+\n+\treturn imageFrameGenerator;\n+}\n+\n+/* Scale the buffers for image frames. */\n+void ImageFrameGenerator::configure(const Size &size)\n+{\n+\t/* Reset the source images to prevent multiple configuration calls */\n+\tscaledFrameDatas_.clear();\n+\tframeCount_ = 0;\n+\tparameter_ = 0;\n+\n+\tfor (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {\n+\t\t/* Scale the imageFrameDatas_ to scaledY and scaledUV */\n+\t\tunsigned int halfSizeWidth = (size.width + 1) / 2;\n+\t\tunsigned int halfSizeHeight = (size.height + 1) / 2;\n+\t\tstd::unique_ptr<uint8_t[]> scaledY =\n+\t\t\tstd::make_unique<uint8_t[]>(size.width * size.height);\n+\t\tstd::unique_ptr<uint8_t[]> scaledUV =\n+\t\t\tstd::make_unique<uint8_t[]>(halfSizeWidth * halfSizeHeight * 2);\n+\t\tauto &src = imageFrameDatas_[i];\n+\n+\t\t/*\n+\t\t * \\todo Some platforms might enforce stride due to GPU, like\n+\t\t * ChromeOS ciri (64). The weight needs to be a multiple of\n+\t\t * the stride to work properly for now.\n+\t\t */\n+\t\tlibyuv::NV12Scale(src.Y.get(), src.size.width,\n+\t\t\t\t src.UV.get(), src.size.width,\n+\t\t\t\t src.size.width, src.size.height,\n+\t\t\t\t scaledY.get(), size.width, scaledUV.get(), size.width,\n+\t\t\t\t size.width, size.height, libyuv::FilterMode::kFilterBilinear);\n+\n+\t\tscaledFrameDatas_.emplace_back(\n+\t\t\tImageFrameData{ std::move(scaledY), std::move(scaledUV), size });\n+\t}\n+}\n+\n+int ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)\n+{\n+\t/* Don't do anything when the list of buffers is empty*/\n+\tASSERT(!scaledFrameDatas_.empty());\n+\n+\tMappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);\n+\n+\tauto planes = mappedFrameBuffer.planes();\n+\n+\t/* Make sure the frameCount does not over the number of images */\n+\tframeCount_ %= imageFrames_->number.value_or(1);\n+\n+\t/* Write the scaledY and scaledUV to the mapped frame buffer */\n+\tlibyuv::NV12Copy(scaledFrameDatas_[frameCount_].Y.get(), size.width,\n+\t\t\t scaledFrameDatas_[frameCount_].UV.get(), size.width, planes[0].begin(),\n+\t\t\t size.width, planes[1].begin(), size.width,\n+\t\t\t size.width, size.height);\n+\n+\t/* Proceed an image every 4 frames */\n+\t/* \\todo Consider setting the proceed interval in the config file */\n+\tparameter_++;\n+\tif (parameter_ % frameInterval == 0)\n+\t\tframeCount_++;\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * \\var ImageFrameGenerator::imageFrameDatas_\n+ * \\brief List of pointers to the not scaled image buffers\n+ */\n+\n+/*\n+ * \\var ImageFrameGenerator::scaledFrameDatas_\n+ * \\brief List of pointers to the scaled image buffers\n+ */\n+\n+/*\n+ * \\var ImageFrameGenerator::imageFrames_\n+ * \\brief Pointer to the imageFrames_ in VirtualCameraData\n+ */\n+\n+/*\n+ * \\var ImageFrameGenerator::parameter_\n+ * \\brief Speed parameter. Change to the next image every parameter_ frames\n+ */\n+\n+} /* namespace libcamera */\ndiff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h\nnew file mode 100644\nindex 00000000..cd243816\n--- /dev/null\n+++ b/src/libcamera/pipeline/virtual/image_frame_generator.h\n@@ -0,0 +1,51 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Google Inc.\n+ *\n+ * image_frame_generator.h - Derived class of FrameGenerator for\n+ * generating frames from images\n+ */\n+\n+#pragma once\n+\n+#include <filesystem>\n+#include <memory>\n+#include <optional>\n+#include <stdint.h>\n+#include <sys/types.h>\n+\n+#include \"frame_generator.h\"\n+\n+namespace libcamera {\n+\n+/* Frame configuration provided by the config file */\n+struct ImageFrames {\n+\tstd::filesystem::path path;\n+\tstd::optional<unsigned int> number;\n+};\n+\n+class ImageFrameGenerator : public FrameGenerator\n+{\n+public:\n+\tstatic std::unique_ptr<ImageFrameGenerator> create(ImageFrames &imageFrames);\n+\n+private:\n+\tstatic constexpr unsigned int frameInterval = 4;\n+\n+\tstruct ImageFrameData {\n+\t\tstd::unique_ptr<uint8_t[]> Y;\n+\t\tstd::unique_ptr<uint8_t[]> UV;\n+\t\tSize size;\n+\t};\n+\n+\tvoid configure(const Size &size) override;\n+\tint generateFrame(const Size &size, const FrameBuffer *buffer) override;\n+\n+\tstd::vector<ImageFrameData> imageFrameDatas_;\n+\tstd::vector<ImageFrameData> scaledFrameDatas_;\n+\tImageFrames *imageFrames_;\n+\tunsigned int frameCount_;\n+\tunsigned int parameter_;\n+};\n+\n+} /* namespace libcamera */\ndiff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build\nindex 0c537777..bb38c193 100644\n--- a/src/libcamera/pipeline/virtual/meson.build\n+++ b/src/libcamera/pipeline/virtual/meson.build\n@@ -1,8 +1,12 @@\n # SPDX-License-Identifier: CC0-1.0\n \n libcamera_internal_sources += files([\n+ 'image_frame_generator.cpp',\n 'test_pattern_generator.cpp',\n 'virtual.cpp',\n ])\n \n+libjpeg = dependency('libjpeg', required : true)\n+\n libcamera_deps += [libyuv_dep]\n+libcamera_deps += [libjpeg]\n", "prefixes": [ "v14", "5/7" ] }