From patchwork Tue Sep 30 12:26:41 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Stefan Klug X-Patchwork-Id: 24519 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id CF00EC328C for ; Tue, 30 Sep 2025 13:15:35 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id E205E6B599; Tue, 30 Sep 2025 15:15:34 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="iW50MnI5"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 2C91462C35 for ; Tue, 30 Sep 2025 15:15:33 +0200 (CEST) Received: from ideasonboard.com (unknown [94.31.94.171]) by perceval.ideasonboard.com (Postfix) with UTF8SMTPSA id CEE01226; Tue, 30 Sep 2025 15:14:04 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1759238044; bh=iaZePcASm/J0ayqJaNJFOLLjY52UvzvyYVvuCKXP1Ws=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=iW50MnI5KYO6r1s07HZUB4uEsxTPfvQRAZ3NPoM8PRqPW5+3NYVoiekWOiQthtrra 3oSfgZO5qvJPQKA8VuiQz1BoFP8DAOQNn4KoaKNKQbtAyuT/wnlDa3W3Hnc+O8Pg9q QHiHS+wiDg2AUteFm+IwLOF0ym9oaShjWU+/vrgM= From: Stefan Klug To: libcamera-devel@lists.libcamera.org Cc: Stefan Klug Subject: [PATCH v1 20/33] libcamera: converter: Add dw100 vertex map class Date: Tue, 30 Sep 2025 14:26:41 +0200 Message-ID: <20250930122726.1837524-21-stefan.klug@ideasonboard.com> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250930122726.1837524-1-stefan.klug@ideasonboard.com> References: <20250930122726.1837524-1-stefan.klug@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Using a custom vertex map the dw100 dewarper is capable of doing complex and useful transformations on the image data. This class implements a pipeline featuring: - Arbitrary ScalerCrop - Full transform support (Flip, 90deg rotations) - Arbitrary move, scale, rotate ScalerCrop and Transform is implemented to provide a interface that is standardized libcamera wide. The rest is implemented on top for more flexible dw100 specific features. Signed-off-by: Stefan Klug --- Changes in v0.9 - Include header in meson.build - Fix black line at top and left when rotation 180 degrees Changes in v0.8 - Cleanup & formatting Changes in v0.5 - Fix crash in std::clamp() due to rounding errors --- .../converter/converter_dw100_vertexmap.h | 80 ++++ .../libcamera/internal/converter/meson.build | 1 + .../converter/converter_dw100_vertexmap.cpp | 362 ++++++++++++++++++ src/libcamera/converter/meson.build | 1 + 4 files changed, 444 insertions(+) create mode 100644 include/libcamera/internal/converter/converter_dw100_vertexmap.h create mode 100644 src/libcamera/converter/converter_dw100_vertexmap.cpp diff --git a/include/libcamera/internal/converter/converter_dw100_vertexmap.h b/include/libcamera/internal/converter/converter_dw100_vertexmap.h new file mode 100644 index 000000000000..9c6e0ffa4513 --- /dev/null +++ b/include/libcamera/internal/converter/converter_dw100_vertexmap.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace libcamera { + +class Dw100VertexMap +{ +public: + enum ScaleMode { + Fill = 0, + Crop = 1, + }; + + void applyLimits(); + void setInputSize(const Size &size) + { + inputSize_ = size; + scalerCrop_ = Rectangle(size); + } + + void setSensorCrop(const Rectangle &rect) { sensorCrop_ = rect; } + + void setScalerCrop(const Rectangle &rect) { scalerCrop_ = rect; } + const Rectangle &effectiveScalerCrop() const { return effectiveScalerCrop_; } + std::pair scalerCropBounds() const + { + return { Rectangle(sensorCrop_.x, sensorCrop_.y, 1, 1), + sensorCrop_ }; + } + + void setOutputSize(const Size &size) { outputSize_ = size; } + const Size &outputSize() const { return outputSize_; } + + void setTransform(const Transform &transform) { transform_ = transform; } + const Transform &transform() const { return transform_; } + + void setScale(const float scale) { scale_ = scale; } + float effectiveScale() const { return (effectiveScaleX_ + effectiveScaleY_) * 0.5; } + + void setRotation(const float rotation) { rotation_ = rotation; } + float rotation() const { return rotation_; } + + void setOffset(const Point &offset) { offset_ = offset; } + const Point &effectiveOffset() const { return effectiveOffset_; } + + void setMode(const ScaleMode mode) { mode_ = mode; } + ScaleMode mode() const { return mode_; } + + std::vector getVertexMap(); + +private: + int getVerticesForLength(const int length); + + Rectangle scalerCrop_; + Rectangle sensorCrop_; + Transform transform_ = Transform::Identity; + Size inputSize_; + Size outputSize_; + Point offset_; + double scale_ = 1.0; + double rotation_ = 0.0; + ScaleMode mode_ = Fill; + double effectiveScaleX_; + double effectiveScaleY_; + Point effectiveOffset_; + Rectangle effectiveScalerCrop_; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/converter/meson.build b/include/libcamera/internal/converter/meson.build index 85007a4b0f8b..128c644cb73f 100644 --- a/include/libcamera/internal/converter/meson.build +++ b/include/libcamera/internal/converter/meson.build @@ -2,5 +2,6 @@ libcamera_internal_headers += files([ 'converter_dw100.h', + 'converter_dw100_vertexmap.h', 'converter_v4l2_m2m.h', ]) diff --git a/src/libcamera/converter/converter_dw100_vertexmap.cpp b/src/libcamera/converter/converter_dw100_vertexmap.cpp new file mode 100644 index 000000000000..10d9e34d98c5 --- /dev/null +++ b/src/libcamera/converter/converter_dw100_vertexmap.cpp @@ -0,0 +1,362 @@ +#include "libcamera/internal/converter/converter_dw100_vertexmap.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "libcamera/internal/vector.h" + +constexpr int kDw100BlockSize = 16; + +namespace libcamera { + +LOG_DECLARE_CATEGORY(Converter) + +using Vector2d = Vector; + +namespace { +void applyTransform(const Transform &t, const double w, const double h, + Vector2d &p) +{ + if (!!(t & Transform::HFlip)) { + p.x() = w - p.x(); + } + + if (!!(t & Transform::VFlip)) { + p.y() = h - p.y(); + } + + if (!!(t & Transform::Transpose)) { + std::swap(p.x(), p.y()); + } +} + +void applyTransform(const Transform &t, Rectangle &s) +{ + if (!!(t & Transform::Transpose)) { + std::swap(s.width, s.height); + } +} + +int roundTowardsZero(double v) +{ + double r = (v < 0) ? std::ceil(v) : std::floor(v); + return static_cast(r); +} + +Vector2d rotatedRectSize(const Vector2d &size, const double rad) +{ + double sa = sin(rad); + double ca = cos(rad); + + return { { std::abs(size.x() * ca) + std::abs(size.y() * sa), + std::abs(size.x() * sa) + std::abs(size.y() * ca) } }; +} + +Vector2d point2Vec2d(const Point &p) +{ + return { { static_cast(p.x), static_cast(p.y) } }; +} + +} /* namespace */ + +/** + * The vertex map class represents a helper for handling dewarper vertex maps. + * There are 3 important sizes in the system: + * + * - The Sensor size. The number of pixels of the whole sensor (Todo specify if + * OB lines are included) + * - The input Rectangle to the dewarper. Describes the pixel data + * flowing into the dewarper in sensor space + * - ScalerCrop rectangle. The rectangle that shall be used for all further + * stages. It is applied after lens dewarping but is in sensor coordinate + * space. + * - The output size. This defines the size, the dewarper should output. + * + * +------------------------+ + * |Sensor size | + * | | + * | +----------------+ | + * | | Input rect | | + * | | | | + * | +----------------+ | + * +------------------------+ + * + * This class implements a vertex map that forms the following pipeline: + * + * +-------------+ +-------------+ +-----------+ +--------------------+ + * | Lens Dewarp | -> | Scaler Crop | -> | Transform | -> | Offset, Scale, Rot | + * +-------------+ +-------------+ +-----------+ +--------------------+ + * + * ToDo: LensDewarp is not yet implemented. An identity map is used instead. + * + * All parameters are clamped to valid values before creating the vertex map. + * + * The constrains process works as follows: + * - The ScalerCrop rectangle is clamped to the input rectangle + * - The ScalerCrop rectangle is transformed by the specified transform + * forming ScalerCropT + * - A rectangle of output size is placed in the center of ScalerCropT + * (OutputRect). + * - Rotate gets applied to OutputRect, + * - Scale is applied, but clamped so that the OutputRect fits completely into + * ScalerCropT (Only regarding dimensions, not position) + * - Offset is clamped so that the OutputRect lies inside ScalerCropT + * + * The lens dewarp map is usually calibrated during tuning and is a map that + * maps from incoming pixels to dewarped pixels. + * + * Dewarp modes: + * - Crop: Fills the output frame with the data from ScalerCropT + * - Preserves aspect ratio + * - Rotate and Offset and Scale are taken into account within the limits. + * - Fill: + * - Fills the output frame with the data from ScalerCropT. + * - Does not preserve aspect ratio + * - Rotate is taken into account + * + */ +void Dw100VertexMap::applyLimits() +{ + int ow = outputSize_.width; + int oh = outputSize_.height; + effectiveScalerCrop_ = scalerCrop_.boundedTo(sensorCrop_); + + /* Map the scalerCrop to the input pixel space */ + Rectangle localScalerCropT = effectiveScalerCrop_.transformedBetween( + sensorCrop_, Rectangle(inputSize_)); + + applyTransform(transform_, localScalerCropT); + + double rad = rotation_ / 180.0 * M_PI; + double sa = sin(rad); + double ca = cos(rad); + + double scale = scale_; + Vector2d offset; + Vector2d size = rotatedRectSize(point2Vec2d(Point(ow, oh)), rad); + Vector2d t; + + /* + * Todo: All these rotations and calculations below are way easier using + * matrices. So reimplement using matrix class. + */ + + /* + * Calculate constraints + */ + if (mode_ == Crop) { + /* Scale up if needed */ + scale = std::max(scale, + std::max(size.x() / localScalerCropT.width, + size.y() / localScalerCropT.height)); + effectiveScaleX_ = scale; + effectiveScaleY_ = scale; + + size = size / scale; + + /* Transform offset to sensor space */ + offset.x() = ca * offset_.x - sa * offset_.y; + offset.y() = sa * offset_.x + ca * offset_.y; + offset = offset / scale; + } else if (mode_ == Fill) { + /* + * Calculate the best x and y scale values to fit the rotated + * localScalerCropT rectangle into the output rectangle. + */ + double diff = (static_cast(localScalerCropT.width) - + localScalerCropT.height) * + 0.5; + double middle = (static_cast(localScalerCropT.width) + + localScalerCropT.height) * + 0.5; + double w = middle + cos(rad * 2) * diff; + double h = middle - cos(rad * 2) * diff; + + size = rotatedRectSize(Vector2d{ { w, h } }, rad); + scale = std::max(size.x() / localScalerCropT.width, + size.y() / localScalerCropT.height); + + effectiveScaleX_ = (ow / w) * scale; + effectiveScaleY_ = (oh / h) * scale; + + size = size / scale; + + t = point2Vec2d(offset_); + t.x() /= effectiveScaleX_; + t.y() /= effectiveScaleY_; + + /* Transform offset to sensor space including local scale */ + offset.x() = ca * t.x() - sa * t.y(); + offset.y() = sa * t.x() + ca * t.y(); + } else { + LOG(Converter, Error) << "Unknown mode " << mode_; + return; + } + + /* Clamp offset in input space. */ + double maxoff; + /* + * Due to rounding errors, size might be slightly bigger than scaler + * crop. Clamp the offset to 0 to prevent a crash in the next clamp. + */ + maxoff = std::max(0.0, (localScalerCropT.width - size.x())) * 0.5; + offset.x() = std::clamp(offset.x(), -maxoff, maxoff); + maxoff = std::max(0.0, (localScalerCropT.height - size.y())) * 0.5; + offset.y() = std::clamp(offset.y(), -maxoff, maxoff); + + /* + * Transform offset back into output space. + * Note the transposed rotation matrix. + */ + t = offset; + offset.x() = ca * t.x() + sa * t.y(); + offset.y() = -sa * t.x() + ca * t.y(); + offset.x() *= effectiveScaleX_; + offset.y() *= effectiveScaleY_; + + effectiveOffset_ = Point(roundTowardsZero(offset.x()), + roundTowardsZero(offset.y())); +} + +std::vector Dw100VertexMap::getVertexMap() +{ + int ow = outputSize_.width; + int oh = outputSize_.height; + int tileCountW = getVerticesForLength(ow); + int tileCountH = getVerticesForLength(oh); + double rad = rotation_ / 180.0 * M_PI; + double sa = sin(rad); + double ca = cos(rad); + + applyLimits(); + + /* + * libcamera handles all crop rectangles in sensor space. But the + * dewarper "sees" only the pixels it gets passed. Note that these might + * not cover exactly the max sensor crop, as there might be a crop + * between ISP and dewarper to crop to a format supported by the + * dewarper. effectiveScalerCrop_ is the crop in sensor space that gets + * fed into the dewarper. localScalerCrop is the sensor crop mapped to + * the data that is fed into the dewarper. + */ + Rectangle localScalerCrop = effectiveScalerCrop_.transformedBetween( + sensorCrop_, Rectangle(inputSize_)); + Rectangle localScalerCropT = localScalerCrop; + applyTransform(transform_, localScalerCropT); + + /* + * The dw100 has a specialty in interpolation that has to be taken into + * account to use in a pixel perfect manner. To explain this, I will + * only use the x direction, the vertical axis behaves the same. + * + * Let's start with a pixel perfect 1:1 mapping of an image with a width + * of 64pixels. The coordinates of the vertex map would then be: + * 0 -- 16 -- 32 -- 48 -- 64 + * Note how the last coordinate lies outside the image (which ends at + * 63) as it is basically the beginning of the next macro block. + * + * if we zoom out a bit we might end up with something like + * -10 -- 0 -- 32 -- 64 -- 74 + * As the dewarper coordinates are unsigned it actually sees + * 0 -- 0 -- 32 -- 64 -- 74 + * Leading to stretched pixels at the beginning and black for everything + * > 63 + * + * Now lets rotate the image by 180 degrees. A trivial rotation would + * end up with: + * + * 64 -- 48 -- 32 -- 16 -- 0 + * + * But as the first column now points to pixel 64 we get a single black + * line. So for a proper 180* rotation, the coordinates need to be + * + * 63 -- 47 -- 31 -- 15 -- -1 + * + * The -1 is clamped to 0 again, leading to a theoretical slight + * interpolation error on the last 16 pixels. + * + * To create this proper transformation there are two things todo: + * + * 1. The rotation centers are offset by -0.5. This evens out for no rotation, + * and leads to a coordinate offset of -1 on 180 degree rotations. + * + * 2. The transformation (flip and transpose) need to act on a width-1 + * to get the same effect. + */ + Vector2d centerS{ { localScalerCropT.width * 0.5 - 0.5, + localScalerCropT.height * 0.5 - 0.5 } }; + Vector2d centerD{ { ow * 0.5 - 0.5, + oh * 0.5 - 0.5 } }; + + LOG(Converter, Debug) + << "Apply vertex map for" + << " inputSize: " << inputSize_ + << " outputSize: " << outputSize_ + << " Transform: " << transformToString(transform_) + << "\n effectiveScalerCrop: " << effectiveScalerCrop_ + << " localScalerCropT: " << localScalerCropT + << " scaleX: " << effectiveScaleX_ + << " scaleY: " << effectiveScaleX_ + << " rotation: " << rotation_ + << " offset: " << effectiveOffset_; + + /* + * For every output tile, calculate the position of the corners in the + * input image. + */ + std::vector res; + res.reserve(tileCountW * tileCountH); + for (int y = 0; y < tileCountH; y++) { + for (int x = 0; x < tileCountW; x++) { + Vector2d p{ { static_cast(x) * kDw100BlockSize, + static_cast(y) * kDw100BlockSize } }; + p = p.max(0.0).min(Vector2d{ { static_cast(ow), + static_cast(oh) } }); + + /* + * Transform into coordinates centered on the output + * rectangle + offset. + */ + p = p - centerD + point2Vec2d(effectiveOffset_); + p.x() /= effectiveScaleX_; + p.y() /= effectiveScaleY_; + + /* Rotate */ + Vector2d p2; + p2.x() = ca * p.x() - sa * p.y(); + p2.y() = sa * p.x() + ca * p.y(); + + /* Transform to localScalerCropT space. */ + p = p2 + centerS; + /* Subtract -1 for the mapping documented above. */ + applyTransform(-transform_, + localScalerCropT.width - 1, + localScalerCropT.height - 1, p); + p.x() += localScalerCrop.x; + p.y() += localScalerCrop.y; + + /* Convert to fixed point */ + uint32_t v = static_cast(p.y() * 16) << 16 | + (static_cast(p.x() * 16) & 0xffff); + res.push_back(v); + } + } + + return res; +} + +int Dw100VertexMap::getVerticesForLength(const int length) +{ + return (length + kDw100BlockSize - 1) / kDw100BlockSize + 1; +} + +} /* namespace libcamera */ diff --git a/src/libcamera/converter/meson.build b/src/libcamera/converter/meson.build index fe2dcebb67da..9f59b57c26b9 100644 --- a/src/libcamera/converter/meson.build +++ b/src/libcamera/converter/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_sources += files([ + 'converter_dw100_vertexmap.cpp', 'converter_dw100.cpp', 'converter_v4l2_m2m.cpp' ])