| Message ID | 20230327122030.11756-10-naush@raspberrypi.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series |
|
| Related | show |
Hi Nick On Mon, Mar 27, 2023 at 01:20:29PM +0100, Naushir Patuck via libcamera-devel wrote: > From: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com> > > Remove any hard-coded assumptions about the target hardware platform > from the autofocus algorithm. Instead, use the "target" string provided > by the camera tuning config and generalised statistics structures to > determing parameters such as grid and region sizes. > > Additionally, PDAF statistics are represented by a generalised region > statistics structure to be device agnostic. > > These changes also require the autofocus algorithm to initialise > region weights on the first frame's prepare()/process() call rather > than during initialisation. > > Signed-off-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com> > Signed-off-by: Naushir Patuck <naush@raspberrypi.com> > Tested-by: Naushir Patuck <naush@raspberrypi.com> I won't pretend I've actually gone through the calculations, but the few comments I had on v1 have been clarified Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com> Thanks j > --- > src/ipa/raspberrypi/cam_helper_imx708.cpp | 23 ++- > src/ipa/raspberrypi/controller/pdaf_data.h | 21 +-- > src/ipa/raspberrypi/controller/rpi/af.cpp | 176 +++++++++++---------- > src/ipa/raspberrypi/controller/rpi/af.h | 28 ++-- > 4 files changed, 133 insertions(+), 115 deletions(-) > > diff --git a/src/ipa/raspberrypi/cam_helper_imx708.cpp b/src/ipa/raspberrypi/cam_helper_imx708.cpp > index 1f213d3c0833..641ba18f4b9d 100644 > --- a/src/ipa/raspberrypi/cam_helper_imx708.cpp > +++ b/src/ipa/raspberrypi/cam_helper_imx708.cpp > @@ -69,11 +69,14 @@ private: > /* Largest long exposure scale factor given as a left shift on the frame length. */ > static constexpr int longExposureShiftMax = 7; > > + static constexpr int pdafStatsRows = 12; > + static constexpr int pdafStatsCols = 16; > + > void populateMetadata(const MdParser::RegisterMap ®isters, > Metadata &metadata) const override; > > static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp, > - PdafData &pdaf); > + PdafRegions &pdaf); > > bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp); > void putAGCStatistics(StatisticsPtr stats); > @@ -120,11 +123,11 @@ void CamHelperImx708::prepare(libcamera::Span<const uint8_t> buffer, Metadata &m > size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3; > > if (buffer.size() > 2 * bytesPerLine) { > - PdafData pdaf; > + PdafRegions pdaf; > if (parsePdafData(&buffer[2 * bytesPerLine], > buffer.size() - 2 * bytesPerLine, > mode_.bitdepth, pdaf)) > - metadata.set("pdaf.data", pdaf); > + metadata.set("pdaf.regions", pdaf); > } > > /* Parse AE-HIST data where present */ > @@ -239,7 +242,7 @@ void CamHelperImx708::populateMetadata(const MdParser::RegisterMap ®isters, > } > > bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, > - unsigned bpp, PdafData &pdaf) > + unsigned bpp, PdafRegions &pdaf) > { > size_t step = bpp >> 1; /* bytes per PDAF grid entry */ > > @@ -248,13 +251,17 @@ bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, > return false; > } > > + pdaf.init({ pdafStatsCols, pdafStatsRows }); > + > ptr += 2 * step; > - for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { > - for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { > + for (unsigned i = 0; i < pdafStatsRows; ++i) { > + for (unsigned j = 0; j < pdafStatsCols; ++j) { > unsigned c = (ptr[0] << 3) | (ptr[1] >> 5); > int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2); > - pdaf.conf[i][j] = c; > - pdaf.phase[i][j] = c ? p : 0; > + PdafData pdafData; > + pdafData.conf = c; > + pdafData.phase = c ? p : 0; > + pdaf.set(libcamera::Point(j, i), { pdafData, 1, 0 }); > ptr += step; > } > } > diff --git a/src/ipa/raspberrypi/controller/pdaf_data.h b/src/ipa/raspberrypi/controller/pdaf_data.h > index 03c00d72c0e8..ae6ab996ded0 100644 > --- a/src/ipa/raspberrypi/controller/pdaf_data.h > +++ b/src/ipa/raspberrypi/controller/pdaf_data.h > @@ -2,20 +2,23 @@ > /* > * Copyright (C) 2022, Raspberry Pi Ltd > * > - * pdaf_data.h - PDAF Metadata; for now this is > - * largely based on IMX708's PDAF "Type 1" output. > + * pdaf_data.h - PDAF Metadata > */ > #pragma once > > #include <stdint.h> > > -#define PDAF_DATA_ROWS 12 > -#define PDAF_DATA_COLS 16 > +#include "region_stats.h" > > -struct PdafData { > - /* Confidence values, in raster order, in arbitrary units */ > - uint16_t conf[PDAF_DATA_ROWS][PDAF_DATA_COLS]; > +namespace RPiController { > > - /* Phase error, in raster order, in s11 Q4 format (S.6.4) */ > - int16_t phase[PDAF_DATA_ROWS][PDAF_DATA_COLS]; > +struct PdafData { > + /* Confidence, in arbitrary units */ > + uint16_t conf; > + /* Phase error, in s16 Q4 format (S.11.4) */ > + int16_t phase; > }; > + > +using PdafRegions = RegionStats<PdafData>; > + > +}; // namespace RPiController > diff --git a/src/ipa/raspberrypi/controller/rpi/af.cpp b/src/ipa/raspberrypi/controller/rpi/af.cpp > index a623651875f2..ed0c8a94d064 100644 > --- a/src/ipa/raspberrypi/controller/rpi/af.cpp > +++ b/src/ipa/raspberrypi/controller/rpi/af.cpp > @@ -174,9 +174,8 @@ Af::Af(Controller *controller) > statsRegion_(0, 0, 0, 0), > windows_(), > useWindows_(false), > - phaseWeights_{}, > - contrastWeights_{}, > - sumWeights_(0), > + phaseWeights_(), > + contrastWeights_(), > scanState_(ScanState::Idle), > initted_(false), > ftarget_(-1.0), > @@ -190,7 +189,15 @@ Af::Af(Controller *controller) > scanData_(), > reportState_(AfState::Idle) > { > - scanData_.reserve(24); > + /* > + * Reserve space for data, to reduce memory fragmentation. It's too early > + * to query the size of the PDAF (from camera) and Contrast (from ISP) > + * statistics, but these are plausible upper bounds. > + */ > + phaseWeights_.w.reserve(16 * 12); > + contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width * > + getHardwareConfig().focusRegions.height); > + scanData_.reserve(32); > } > > Af::~Af() > @@ -226,7 +233,7 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met > << statsRegion_.y << ',' > << statsRegion_.width << ',' > << statsRegion_.height; > - computeWeights(); > + invalidateWeights(); > > if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { > /* > @@ -239,111 +246,99 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met > skipCount_ = cfg_.skipFrames; > } > > -void Af::computeWeights() > +void Af::computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols) > { > - constexpr int MaxCellWeight = 240 / (int)MaxWindows; > + wgts->rows = rows; > + wgts->cols = cols; > + wgts->sum = 0; > + wgts->w.resize(rows * cols); > + std::fill(wgts->w.begin(), wgts->w.end(), 0); > > - sumWeights_ = 0; > - for (int i = 0; i < PDAF_DATA_ROWS; ++i) > - std::fill(phaseWeights_[i], phaseWeights_[i] + PDAF_DATA_COLS, 0); > - > - if (useWindows_ && > - statsRegion_.width >= PDAF_DATA_COLS && statsRegion_.height >= PDAF_DATA_ROWS) { > + if (rows > 0 && cols > 0 && useWindows_ && > + statsRegion_.height >= rows && statsRegion_.width >= cols) { > /* > * Here we just merge all of the given windows, weighted by area. > * \todo Perhaps a better approach might be to find the phase in each > * window and choose either the closest or the highest-confidence one? > - * > - * Using mostly "int" arithmetic, because Rectangle has signed x, y > + * Ensure weights sum to less than (1<<16). 46080 is a "round number" > + * below 65536, for better rounding when window size is a simple > + * fraction of image dimensions. > */ > - int cellH = (int)(statsRegion_.height / PDAF_DATA_ROWS); > - int cellW = (int)(statsRegion_.width / PDAF_DATA_COLS); > - int cellA = cellH * cellW; > + const unsigned maxCellWeight = 46080u / (MaxWindows * rows * cols); > + const unsigned cellH = statsRegion_.height / rows; > + const unsigned cellW = statsRegion_.width / cols; > + const unsigned cellA = cellH * cellW; > > for (auto &w : windows_) { > - for (int i = 0; i < PDAF_DATA_ROWS; ++i) { > - int y0 = std::max(statsRegion_.y + cellH * i, w.y); > - int y1 = std::min(statsRegion_.y + cellH * (i + 1), w.y + (int)(w.height)); > + for (unsigned r = 0; r < rows; ++r) { > + int y0 = std::max(statsRegion_.y + (int)(cellH * r), w.y); > + int y1 = std::min(statsRegion_.y + (int)(cellH * (r + 1)), > + w.y + (int)(w.height)); > if (y0 >= y1) > continue; > y1 -= y0; > - for (int j = 0; j < PDAF_DATA_COLS; ++j) { > - int x0 = std::max(statsRegion_.x + cellW * j, w.x); > - int x1 = std::min(statsRegion_.x + cellW * (j + 1), w.x + (int)(w.width)); > + for (unsigned c = 0; c < cols; ++c) { > + int x0 = std::max(statsRegion_.x + (int)(cellW * c), w.x); > + int x1 = std::min(statsRegion_.x + (int)(cellW * (c + 1)), > + w.x + (int)(w.width)); > if (x0 >= x1) > continue; > - int a = y1 * (x1 - x0); > - a = (MaxCellWeight * a + cellA - 1) / cellA; > - phaseWeights_[i][j] += a; > - sumWeights_ += a; > + unsigned a = y1 * (x1 - x0); > + a = (maxCellWeight * a + cellA - 1) / cellA; > + wgts->w[r * cols + c] += a; > + wgts->sum += a; > } > } > } > } > > - if (sumWeights_ == 0) { > - /* > - * Default AF window is the middle 1/2 width of the middle 1/3 height > - * since this maps nicely to both PDAF (16x12) and Focus (4x3) grids. > - */ > - for (int i = PDAF_DATA_ROWS / 3; i < 2 * PDAF_DATA_ROWS / 3; ++i) { > - for (int j = PDAF_DATA_COLS / 4; j < 3 * PDAF_DATA_COLS / 4; ++j) { > - phaseWeights_[i][j] = MaxCellWeight; > - sumWeights_ += MaxCellWeight; > + if (wgts->sum == 0) { > + /* Default AF window is the middle 1/2 width of the middle 1/3 height */ > + for (unsigned r = rows / 3; r < rows - rows / 3; ++r) { > + for (unsigned c = cols / 4; c < cols - cols / 4; ++c) { > + wgts->w[r * cols + c] = 1; > + wgts->sum += 1; > } > } > } > +} > > - /* Scale from PDAF to Focus Statistics grid (which has fixed size 4x3) */ > - constexpr int FocusStatsRows = 3; > - constexpr int FocusStatsCols = 4; > - static_assert(FOCUS_REGIONS == FocusStatsRows * FocusStatsCols); > - static_assert(PDAF_DATA_ROWS % FocusStatsRows == 0); > - static_assert(PDAF_DATA_COLS % FocusStatsCols == 0); > - constexpr int YFactor = PDAF_DATA_ROWS / FocusStatsRows; > - constexpr int XFactor = PDAF_DATA_COLS / FocusStatsCols; > - > - LOG(RPiAf, Debug) << "Recomputed weights:"; > - for (int i = 0; i < FocusStatsRows; ++i) { > - for (int j = 0; j < FocusStatsCols; ++j) { > - unsigned w = 0; > - for (int y = 0; y < YFactor; ++y) > - for (int x = 0; x < XFactor; ++x) > - w += phaseWeights_[YFactor * i + y][XFactor * j + x]; > - contrastWeights_[FocusStatsCols * i + j] = w; > - } > - LOG(RPiAf, Debug) << " " > - << contrastWeights_[FocusStatsCols * i + 0] << " " > - << contrastWeights_[FocusStatsCols * i + 1] << " " > - << contrastWeights_[FocusStatsCols * i + 2] << " " > - << contrastWeights_[FocusStatsCols * i + 3]; > - } > +void Af::invalidateWeights() > +{ > + phaseWeights_.sum = 0; > + contrastWeights_.sum = 0; > } > > -bool Af::getPhase(PdafData const &data, double &phase, double &conf) const > +bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) > { > + libcamera::Size size = regions.size(); > + if (size.height != phaseWeights_.rows || size.width != phaseWeights_.cols || > + phaseWeights_.sum == 0) { > + LOG(RPiAf, Debug) << "Recompute Phase weights " << size.width << 'x' << size.height; > + computeWeights(&phaseWeights_, size.height, size.width); > + } > + > uint32_t sumWc = 0; > int64_t sumWcp = 0; > - > - for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { > - for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { > - if (phaseWeights_[i][j]) { > - uint32_t c = data.conf[i][j]; > - if (c >= cfg_.confThresh) { > - if (c > cfg_.confClip) > - c = cfg_.confClip; > - c -= (cfg_.confThresh >> 2); > - sumWc += phaseWeights_[i][j] * c; > - c -= (cfg_.confThresh >> 2); > - sumWcp += phaseWeights_[i][j] * data.phase[i][j] * (int64_t)c; > - } > + for (unsigned i = 0; i < regions.numRegions(); ++i) { > + unsigned w = phaseWeights_.w[i]; > + if (w) { > + const PdafData &data = regions.get(i).val; > + unsigned c = data.conf; > + if (c >= cfg_.confThresh) { > + if (c > cfg_.confClip) > + c = cfg_.confClip; > + c -= (cfg_.confThresh >> 2); > + sumWc += w * c; > + c -= (cfg_.confThresh >> 2); > + sumWcp += (int64_t)(w * c) * (int64_t)data.phase; > } > } > } > > - if (0 < sumWeights_ && sumWeights_ <= sumWc) { > + if (0 < phaseWeights_.sum && phaseWeights_.sum <= sumWc) { > phase = (double)sumWcp / (double)sumWc; > - conf = (double)sumWc / (double)sumWeights_; > + conf = (double)sumWc / (double)phaseWeights_.sum; > return true; > } else { > phase = 0.0; > @@ -352,14 +347,21 @@ bool Af::getPhase(PdafData const &data, double &phase, double &conf) const > } > } > > -double Af::getContrast(const FocusRegions &focusStats) const > +double Af::getContrast(const FocusRegions &focusStats) > { > - uint32_t sumWc = 0; > + libcamera::Size size = focusStats.size(); > + if (size.height != contrastWeights_.rows || > + size.width != contrastWeights_.cols || contrastWeights_.sum == 0) { > + LOG(RPiAf, Debug) << "Recompute Contrast weights " > + << size.width << 'x' << size.height; > + computeWeights(&contrastWeights_, size.height, size.width); > + } > > + uint64_t sumWc = 0; > for (unsigned i = 0; i < focusStats.numRegions(); ++i) > - sumWc += contrastWeights_[i] * focusStats.get(i).val; > + sumWc += contrastWeights_.w[i] * focusStats.get(i).val; > > - return (sumWeights_ == 0) ? 0.0 : (double)sumWc / (double)sumWeights_; > + return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0; > } > > void Af::doPDAF(double phase, double conf) > @@ -623,14 +625,14 @@ void Af::prepare(Metadata *imageMetadata) > > if (initted_) { > /* Get PDAF from the embedded metadata, and run AF algorithm core */ > - PdafData data; > + PdafRegions regions; > double phase = 0.0, conf = 0.0; > double oldFt = ftarget_; > double oldFs = fsmooth_; > ScanState oldSs = scanState_; > uint32_t oldSt = stepCount_; > - if (imageMetadata->get("pdaf.data", data) == 0) > - getPhase(data, phase, conf); > + if (imageMetadata->get("pdaf.regions", regions) == 0) > + getPhase(regions, phase, conf); > doAF(prevContrast_, phase, conf); > updateLensPosition(); > LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) > @@ -691,7 +693,7 @@ void Af::setMetering(bool mode) > { > if (useWindows_ != mode) { > useWindows_ = mode; > - computeWeights(); > + invalidateWeights(); > } > } > > @@ -708,7 +710,9 @@ void Af::setWindows(libcamera::Span<libcamera::Rectangle const> const &wins) > if (windows_.size() >= MaxWindows) > break; > } > - computeWeights(); > + > + if (useWindows_) > + invalidateWeights(); > } > > bool Af::setLensPosition(double dioptres, int *hwpos) > diff --git a/src/ipa/raspberrypi/controller/rpi/af.h b/src/ipa/raspberrypi/controller/rpi/af.h > index 7959371baf64..b479feb88c39 100644 > --- a/src/ipa/raspberrypi/controller/rpi/af.h > +++ b/src/ipa/raspberrypi/controller/rpi/af.h > @@ -11,12 +11,6 @@ > #include "../pdaf_data.h" > #include "../pwl.h" > > -/* > - * \todo FOCUS_REGIONS is taken from bcm2835-isp.h, but should be made as a > - * generic RegionStats structure. > - */ > -#define FOCUS_REGIONS 12 > - > /* > * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. > * > @@ -121,9 +115,20 @@ private: > double conf; > }; > > - void computeWeights(); > - bool getPhase(PdafData const &data, double &phase, double &conf) const; > - double getContrast(const FocusRegions &focusStats) const; > + struct RegionWeights { > + unsigned rows; > + unsigned cols; > + uint32_t sum; > + std::vector<uint16_t> w; > + > + RegionWeights() > + : rows(0), cols(0), sum(0), w() {} > + }; > + > + void computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols); > + void invalidateWeights(); > + bool getPhase(PdafRegions const ®ions, double &phase, double &conf); > + double getContrast(const FocusRegions &focusStats); > void doPDAF(double phase, double conf); > bool earlyTerminationByPhase(double phase); > double findPeak(unsigned index) const; > @@ -143,9 +148,8 @@ private: > libcamera::Rectangle statsRegion_; > std::vector<libcamera::Rectangle> windows_; > bool useWindows_; > - uint8_t phaseWeights_[PDAF_DATA_ROWS][PDAF_DATA_COLS]; > - uint16_t contrastWeights_[FOCUS_REGIONS]; > - uint32_t sumWeights_; > + RegionWeights phaseWeights_; > + RegionWeights contrastWeights_; > > /* Working state. */ > ScanState scanState_; > -- > 2.34.1 >
diff --git a/src/ipa/raspberrypi/cam_helper_imx708.cpp b/src/ipa/raspberrypi/cam_helper_imx708.cpp index 1f213d3c0833..641ba18f4b9d 100644 --- a/src/ipa/raspberrypi/cam_helper_imx708.cpp +++ b/src/ipa/raspberrypi/cam_helper_imx708.cpp @@ -69,11 +69,14 @@ private: /* Largest long exposure scale factor given as a left shift on the frame length. */ static constexpr int longExposureShiftMax = 7; + static constexpr int pdafStatsRows = 12; + static constexpr int pdafStatsCols = 16; + void populateMetadata(const MdParser::RegisterMap ®isters, Metadata &metadata) const override; static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp, - PdafData &pdaf); + PdafRegions &pdaf); bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp); void putAGCStatistics(StatisticsPtr stats); @@ -120,11 +123,11 @@ void CamHelperImx708::prepare(libcamera::Span<const uint8_t> buffer, Metadata &m size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3; if (buffer.size() > 2 * bytesPerLine) { - PdafData pdaf; + PdafRegions pdaf; if (parsePdafData(&buffer[2 * bytesPerLine], buffer.size() - 2 * bytesPerLine, mode_.bitdepth, pdaf)) - metadata.set("pdaf.data", pdaf); + metadata.set("pdaf.regions", pdaf); } /* Parse AE-HIST data where present */ @@ -239,7 +242,7 @@ void CamHelperImx708::populateMetadata(const MdParser::RegisterMap ®isters, } bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, - unsigned bpp, PdafData &pdaf) + unsigned bpp, PdafRegions &pdaf) { size_t step = bpp >> 1; /* bytes per PDAF grid entry */ @@ -248,13 +251,17 @@ bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, return false; } + pdaf.init({ pdafStatsCols, pdafStatsRows }); + ptr += 2 * step; - for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { - for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { + for (unsigned i = 0; i < pdafStatsRows; ++i) { + for (unsigned j = 0; j < pdafStatsCols; ++j) { unsigned c = (ptr[0] << 3) | (ptr[1] >> 5); int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2); - pdaf.conf[i][j] = c; - pdaf.phase[i][j] = c ? p : 0; + PdafData pdafData; + pdafData.conf = c; + pdafData.phase = c ? p : 0; + pdaf.set(libcamera::Point(j, i), { pdafData, 1, 0 }); ptr += step; } } diff --git a/src/ipa/raspberrypi/controller/pdaf_data.h b/src/ipa/raspberrypi/controller/pdaf_data.h index 03c00d72c0e8..ae6ab996ded0 100644 --- a/src/ipa/raspberrypi/controller/pdaf_data.h +++ b/src/ipa/raspberrypi/controller/pdaf_data.h @@ -2,20 +2,23 @@ /* * Copyright (C) 2022, Raspberry Pi Ltd * - * pdaf_data.h - PDAF Metadata; for now this is - * largely based on IMX708's PDAF "Type 1" output. + * pdaf_data.h - PDAF Metadata */ #pragma once #include <stdint.h> -#define PDAF_DATA_ROWS 12 -#define PDAF_DATA_COLS 16 +#include "region_stats.h" -struct PdafData { - /* Confidence values, in raster order, in arbitrary units */ - uint16_t conf[PDAF_DATA_ROWS][PDAF_DATA_COLS]; +namespace RPiController { - /* Phase error, in raster order, in s11 Q4 format (S.6.4) */ - int16_t phase[PDAF_DATA_ROWS][PDAF_DATA_COLS]; +struct PdafData { + /* Confidence, in arbitrary units */ + uint16_t conf; + /* Phase error, in s16 Q4 format (S.11.4) */ + int16_t phase; }; + +using PdafRegions = RegionStats<PdafData>; + +}; // namespace RPiController diff --git a/src/ipa/raspberrypi/controller/rpi/af.cpp b/src/ipa/raspberrypi/controller/rpi/af.cpp index a623651875f2..ed0c8a94d064 100644 --- a/src/ipa/raspberrypi/controller/rpi/af.cpp +++ b/src/ipa/raspberrypi/controller/rpi/af.cpp @@ -174,9 +174,8 @@ Af::Af(Controller *controller) statsRegion_(0, 0, 0, 0), windows_(), useWindows_(false), - phaseWeights_{}, - contrastWeights_{}, - sumWeights_(0), + phaseWeights_(), + contrastWeights_(), scanState_(ScanState::Idle), initted_(false), ftarget_(-1.0), @@ -190,7 +189,15 @@ Af::Af(Controller *controller) scanData_(), reportState_(AfState::Idle) { - scanData_.reserve(24); + /* + * Reserve space for data, to reduce memory fragmentation. It's too early + * to query the size of the PDAF (from camera) and Contrast (from ISP) + * statistics, but these are plausible upper bounds. + */ + phaseWeights_.w.reserve(16 * 12); + contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width * + getHardwareConfig().focusRegions.height); + scanData_.reserve(32); } Af::~Af() @@ -226,7 +233,7 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met << statsRegion_.y << ',' << statsRegion_.width << ',' << statsRegion_.height; - computeWeights(); + invalidateWeights(); if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { /* @@ -239,111 +246,99 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met skipCount_ = cfg_.skipFrames; } -void Af::computeWeights() +void Af::computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols) { - constexpr int MaxCellWeight = 240 / (int)MaxWindows; + wgts->rows = rows; + wgts->cols = cols; + wgts->sum = 0; + wgts->w.resize(rows * cols); + std::fill(wgts->w.begin(), wgts->w.end(), 0); - sumWeights_ = 0; - for (int i = 0; i < PDAF_DATA_ROWS; ++i) - std::fill(phaseWeights_[i], phaseWeights_[i] + PDAF_DATA_COLS, 0); - - if (useWindows_ && - statsRegion_.width >= PDAF_DATA_COLS && statsRegion_.height >= PDAF_DATA_ROWS) { + if (rows > 0 && cols > 0 && useWindows_ && + statsRegion_.height >= rows && statsRegion_.width >= cols) { /* * Here we just merge all of the given windows, weighted by area. * \todo Perhaps a better approach might be to find the phase in each * window and choose either the closest or the highest-confidence one? - * - * Using mostly "int" arithmetic, because Rectangle has signed x, y + * Ensure weights sum to less than (1<<16). 46080 is a "round number" + * below 65536, for better rounding when window size is a simple + * fraction of image dimensions. */ - int cellH = (int)(statsRegion_.height / PDAF_DATA_ROWS); - int cellW = (int)(statsRegion_.width / PDAF_DATA_COLS); - int cellA = cellH * cellW; + const unsigned maxCellWeight = 46080u / (MaxWindows * rows * cols); + const unsigned cellH = statsRegion_.height / rows; + const unsigned cellW = statsRegion_.width / cols; + const unsigned cellA = cellH * cellW; for (auto &w : windows_) { - for (int i = 0; i < PDAF_DATA_ROWS; ++i) { - int y0 = std::max(statsRegion_.y + cellH * i, w.y); - int y1 = std::min(statsRegion_.y + cellH * (i + 1), w.y + (int)(w.height)); + for (unsigned r = 0; r < rows; ++r) { + int y0 = std::max(statsRegion_.y + (int)(cellH * r), w.y); + int y1 = std::min(statsRegion_.y + (int)(cellH * (r + 1)), + w.y + (int)(w.height)); if (y0 >= y1) continue; y1 -= y0; - for (int j = 0; j < PDAF_DATA_COLS; ++j) { - int x0 = std::max(statsRegion_.x + cellW * j, w.x); - int x1 = std::min(statsRegion_.x + cellW * (j + 1), w.x + (int)(w.width)); + for (unsigned c = 0; c < cols; ++c) { + int x0 = std::max(statsRegion_.x + (int)(cellW * c), w.x); + int x1 = std::min(statsRegion_.x + (int)(cellW * (c + 1)), + w.x + (int)(w.width)); if (x0 >= x1) continue; - int a = y1 * (x1 - x0); - a = (MaxCellWeight * a + cellA - 1) / cellA; - phaseWeights_[i][j] += a; - sumWeights_ += a; + unsigned a = y1 * (x1 - x0); + a = (maxCellWeight * a + cellA - 1) / cellA; + wgts->w[r * cols + c] += a; + wgts->sum += a; } } } } - if (sumWeights_ == 0) { - /* - * Default AF window is the middle 1/2 width of the middle 1/3 height - * since this maps nicely to both PDAF (16x12) and Focus (4x3) grids. - */ - for (int i = PDAF_DATA_ROWS / 3; i < 2 * PDAF_DATA_ROWS / 3; ++i) { - for (int j = PDAF_DATA_COLS / 4; j < 3 * PDAF_DATA_COLS / 4; ++j) { - phaseWeights_[i][j] = MaxCellWeight; - sumWeights_ += MaxCellWeight; + if (wgts->sum == 0) { + /* Default AF window is the middle 1/2 width of the middle 1/3 height */ + for (unsigned r = rows / 3; r < rows - rows / 3; ++r) { + for (unsigned c = cols / 4; c < cols - cols / 4; ++c) { + wgts->w[r * cols + c] = 1; + wgts->sum += 1; } } } +} - /* Scale from PDAF to Focus Statistics grid (which has fixed size 4x3) */ - constexpr int FocusStatsRows = 3; - constexpr int FocusStatsCols = 4; - static_assert(FOCUS_REGIONS == FocusStatsRows * FocusStatsCols); - static_assert(PDAF_DATA_ROWS % FocusStatsRows == 0); - static_assert(PDAF_DATA_COLS % FocusStatsCols == 0); - constexpr int YFactor = PDAF_DATA_ROWS / FocusStatsRows; - constexpr int XFactor = PDAF_DATA_COLS / FocusStatsCols; - - LOG(RPiAf, Debug) << "Recomputed weights:"; - for (int i = 0; i < FocusStatsRows; ++i) { - for (int j = 0; j < FocusStatsCols; ++j) { - unsigned w = 0; - for (int y = 0; y < YFactor; ++y) - for (int x = 0; x < XFactor; ++x) - w += phaseWeights_[YFactor * i + y][XFactor * j + x]; - contrastWeights_[FocusStatsCols * i + j] = w; - } - LOG(RPiAf, Debug) << " " - << contrastWeights_[FocusStatsCols * i + 0] << " " - << contrastWeights_[FocusStatsCols * i + 1] << " " - << contrastWeights_[FocusStatsCols * i + 2] << " " - << contrastWeights_[FocusStatsCols * i + 3]; - } +void Af::invalidateWeights() +{ + phaseWeights_.sum = 0; + contrastWeights_.sum = 0; } -bool Af::getPhase(PdafData const &data, double &phase, double &conf) const +bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) { + libcamera::Size size = regions.size(); + if (size.height != phaseWeights_.rows || size.width != phaseWeights_.cols || + phaseWeights_.sum == 0) { + LOG(RPiAf, Debug) << "Recompute Phase weights " << size.width << 'x' << size.height; + computeWeights(&phaseWeights_, size.height, size.width); + } + uint32_t sumWc = 0; int64_t sumWcp = 0; - - for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { - for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { - if (phaseWeights_[i][j]) { - uint32_t c = data.conf[i][j]; - if (c >= cfg_.confThresh) { - if (c > cfg_.confClip) - c = cfg_.confClip; - c -= (cfg_.confThresh >> 2); - sumWc += phaseWeights_[i][j] * c; - c -= (cfg_.confThresh >> 2); - sumWcp += phaseWeights_[i][j] * data.phase[i][j] * (int64_t)c; - } + for (unsigned i = 0; i < regions.numRegions(); ++i) { + unsigned w = phaseWeights_.w[i]; + if (w) { + const PdafData &data = regions.get(i).val; + unsigned c = data.conf; + if (c >= cfg_.confThresh) { + if (c > cfg_.confClip) + c = cfg_.confClip; + c -= (cfg_.confThresh >> 2); + sumWc += w * c; + c -= (cfg_.confThresh >> 2); + sumWcp += (int64_t)(w * c) * (int64_t)data.phase; } } } - if (0 < sumWeights_ && sumWeights_ <= sumWc) { + if (0 < phaseWeights_.sum && phaseWeights_.sum <= sumWc) { phase = (double)sumWcp / (double)sumWc; - conf = (double)sumWc / (double)sumWeights_; + conf = (double)sumWc / (double)phaseWeights_.sum; return true; } else { phase = 0.0; @@ -352,14 +347,21 @@ bool Af::getPhase(PdafData const &data, double &phase, double &conf) const } } -double Af::getContrast(const FocusRegions &focusStats) const +double Af::getContrast(const FocusRegions &focusStats) { - uint32_t sumWc = 0; + libcamera::Size size = focusStats.size(); + if (size.height != contrastWeights_.rows || + size.width != contrastWeights_.cols || contrastWeights_.sum == 0) { + LOG(RPiAf, Debug) << "Recompute Contrast weights " + << size.width << 'x' << size.height; + computeWeights(&contrastWeights_, size.height, size.width); + } + uint64_t sumWc = 0; for (unsigned i = 0; i < focusStats.numRegions(); ++i) - sumWc += contrastWeights_[i] * focusStats.get(i).val; + sumWc += contrastWeights_.w[i] * focusStats.get(i).val; - return (sumWeights_ == 0) ? 0.0 : (double)sumWc / (double)sumWeights_; + return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0; } void Af::doPDAF(double phase, double conf) @@ -623,14 +625,14 @@ void Af::prepare(Metadata *imageMetadata) if (initted_) { /* Get PDAF from the embedded metadata, and run AF algorithm core */ - PdafData data; + PdafRegions regions; double phase = 0.0, conf = 0.0; double oldFt = ftarget_; double oldFs = fsmooth_; ScanState oldSs = scanState_; uint32_t oldSt = stepCount_; - if (imageMetadata->get("pdaf.data", data) == 0) - getPhase(data, phase, conf); + if (imageMetadata->get("pdaf.regions", regions) == 0) + getPhase(regions, phase, conf); doAF(prevContrast_, phase, conf); updateLensPosition(); LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) @@ -691,7 +693,7 @@ void Af::setMetering(bool mode) { if (useWindows_ != mode) { useWindows_ = mode; - computeWeights(); + invalidateWeights(); } } @@ -708,7 +710,9 @@ void Af::setWindows(libcamera::Span<libcamera::Rectangle const> const &wins) if (windows_.size() >= MaxWindows) break; } - computeWeights(); + + if (useWindows_) + invalidateWeights(); } bool Af::setLensPosition(double dioptres, int *hwpos) diff --git a/src/ipa/raspberrypi/controller/rpi/af.h b/src/ipa/raspberrypi/controller/rpi/af.h index 7959371baf64..b479feb88c39 100644 --- a/src/ipa/raspberrypi/controller/rpi/af.h +++ b/src/ipa/raspberrypi/controller/rpi/af.h @@ -11,12 +11,6 @@ #include "../pdaf_data.h" #include "../pwl.h" -/* - * \todo FOCUS_REGIONS is taken from bcm2835-isp.h, but should be made as a - * generic RegionStats structure. - */ -#define FOCUS_REGIONS 12 - /* * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. * @@ -121,9 +115,20 @@ private: double conf; }; - void computeWeights(); - bool getPhase(PdafData const &data, double &phase, double &conf) const; - double getContrast(const FocusRegions &focusStats) const; + struct RegionWeights { + unsigned rows; + unsigned cols; + uint32_t sum; + std::vector<uint16_t> w; + + RegionWeights() + : rows(0), cols(0), sum(0), w() {} + }; + + void computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols); + void invalidateWeights(); + bool getPhase(PdafRegions const ®ions, double &phase, double &conf); + double getContrast(const FocusRegions &focusStats); void doPDAF(double phase, double conf); bool earlyTerminationByPhase(double phase); double findPeak(unsigned index) const; @@ -143,9 +148,8 @@ private: libcamera::Rectangle statsRegion_; std::vector<libcamera::Rectangle> windows_; bool useWindows_; - uint8_t phaseWeights_[PDAF_DATA_ROWS][PDAF_DATA_COLS]; - uint16_t contrastWeights_[FOCUS_REGIONS]; - uint32_t sumWeights_; + RegionWeights phaseWeights_; + RegionWeights contrastWeights_; /* Working state. */ ScanState scanState_;