@@ -64,9 +64,8 @@ class Quantization(enum.Enum):
LIMITED = 1
-def scale_coeff(coeff, quantization, luma, precision, invert):
- """Scale a coefficient to the output range dictated by the quantization and
- the precision.
+def scale_coeff(coeff, quantization, luma):
+ """Scale a coefficient to the output range dictated by the quantization.
Parameters
----------
@@ -76,9 +75,6 @@ def scale_coeff(coeff, quantization, luma, precision, invert):
The quantization, either FULL or LIMITED
luma : bool
True if the coefficient corresponds to a luma value, False otherwise
- precision : int
- The desired precision for the scaled coefficient as a number of
- fractional bits
"""
# Assume the input range is 8 bits. The output range is set by the
@@ -92,10 +88,7 @@ def scale_coeff(coeff, quantization, luma, precision, invert):
else:
out_range = 240 - 16
- if invert:
- in_range, out_range = out_range, in_range
-
- return coeff * out_range / in_range * (1 << precision)
+ return coeff * out_range / in_range
def round_array(values):
@@ -172,20 +165,21 @@ def main(argv):
encoding = encodings[args.encoding]
quantization = Quantization[args.quantization.upper()]
- if args.invert:
- encoding = np.linalg.inv(encoding)
-
# Scale and round the encoding coefficients based on the precision and
# quantization range.
luma = True
scaled_coeffs = []
for line in encoding:
- line = [scale_coeff(coeff, quantization, luma, precision.fractional, args.invert) for coeff in line]
+ line = [scale_coeff(coeff, quantization, luma) for coeff in line]
scaled_coeffs.append(line)
luma = False
+ if args.invert:
+ scaled_coeffs = np.linalg.inv(scaled_coeffs)
+
rounded_coeffs = []
for line in scaled_coeffs:
+ line = [coeff * (1 << precision.fractional) for coeff in line]
line = round_array(line)
# Convert coefficients to the number of bits selected by the precision.
Coefficient scaling in the inverted CSC table is wrong, as the luma-related coefficients are not in the first row, but in the first column. The simplest way to fix it is to remove the inversion handling from scale_coeff() and invert the matrix after scaling the coefficients. The multiplication by the fixed point precision must still happen after inversion, and must thus be moved out of scale_coeff(). Fixes: b5c4b6e0339a ("utils: rkisp1: gen-csc-table: Add support for inverting the CSC") Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> --- utils/rkisp1/gen-csc-table.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-)