From patchwork Wed May 13 09:11:19 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Naushir Patuck X-Patchwork-Id: 3786 Return-Path: Received: from mail-wm1-x336.google.com (mail-wm1-x336.google.com [IPv6:2a00:1450:4864:20::336]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 80C4C60DFC for ; Wed, 13 May 2020 11:12:43 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (2048-bit key; unprotected) header.d=raspberrypi.com header.i=@raspberrypi.com header.b="DMYn3uBs"; dkim-atps=neutral Received: by mail-wm1-x336.google.com with SMTP id f134so13553114wmf.1 for ; Wed, 13 May 2020 02:12:43 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=raspberrypi.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=7nSYnRG0NQEsGjzIG8Uc9rfvJsxYjJNhMbep/AHEk3c=; b=DMYn3uBsvpC88kX4UPXzHeO0vNd2KUeJsf9f62ZQ1/RCrnQWaIukgNmXY2po+ceznx 6LOLhcDXGslLiGfK0WRsvHckkelmWxiD5Yh8SoisSywmd4wcn19eqwWTJBvuauvMm76Z o9r1WG/E6y2ORcAFm9qSCjQyzZA/H+Xt0UhD+UDw/siMub1wx5fLViKsubQcpWI8qrL9 DmHOdBXXPczciG7MX+FRfNyD1m+4VqMrFNipovVnWjRPRaXg/tbzwFjZ9kd8CEJmv5fo lr/cplhDTZdLm8XNT++to/eEKh0rRUoYaC4+iun89bym6mWg+LBszvWaAc8CPX/F/CdG xY8w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=7nSYnRG0NQEsGjzIG8Uc9rfvJsxYjJNhMbep/AHEk3c=; b=NveQ9nuSRNFG4sfIEIWI4AOjItsW9QxkB52VsywtEDV3xYhIiWlef3tOl48fxcRXTK B0QknX41tR+FrxB5PC++y1OdrZCAzSfExoNJq2Nh+IubVcUiUHd0kmRXG9RiRpHsBdp+ 3wVMiP8skGD8JZn69mF9uc5llMnp0Rtt7T8gzS6QAdeIWcUu17y3kYkqaHgodxewDW/j dElfRPOhUrELZ3V33YPb2JX7sEvaT+rG11/3V1bZY+OFNfo8xu0DIQTH+vkAWqmGSm3z yoZJFDBVzsw7M/aaM12kD5ggOm1Ca0nMHt8NKPLEZBpamZX8Wyy1deYTFb6+vYmqLHz7 ajZw== X-Gm-Message-State: AGi0PubDS7gkOT4WUwUZjbqoTpOn8gDAJzWsDOyajF1qvC1+J0XPLoVv 8u2ZS2xTh8bkWRDAtQJp9LjhBXrnvre6xQ== X-Google-Smtp-Source: APiQypIbWsJfPrjClrmYYjDIK7kd0LiwbMPVN57Tj/0V5n9CCBZfH1mxUV3WJk1ikbvo/WA+LsrYLA== X-Received: by 2002:a1c:208a:: with SMTP id g132mr20063282wmg.78.1589361162413; Wed, 13 May 2020 02:12:42 -0700 (PDT) Received: from naushir-VirtualBox.patuck.local ([88.97.76.4]) by smtp.gmail.com with ESMTPSA id n128sm8500184wmb.47.2020.05.13.02.12.41 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 13 May 2020 02:12:41 -0700 (PDT) From: Naushir Patuck To: libcamera-devel@lists.libcamera.org Date: Wed, 13 May 2020 10:11:19 +0100 Message-Id: <20200513091120.1653-3-naush@raspberrypi.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200513091120.1653-1-naush@raspberrypi.com> References: <20200513091120.1653-1-naush@raspberrypi.com> Subject: [libcamera-devel] [PATCH v2 2/3] libcamera: raspberrypi: Add control of sensor vblanking X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 13 May 2020 09:12:44 -0000 Add support for setting V4L2_CID_VBLANK appropriately when setting V4L2_CID_EXPOSURE. This will allow adaptive framerates during viewfinder use cases (e.g. when the exposure time goes above 33ms, we can reduce the framerate to lower than 30fps). The minimum and maximum frame durations are provided via libcamera controls, and will prioritise exposure time limits over any AGC request. V4L2_CID_VBLANK is controlled through the staggered writer, just like the exposure and gain controls. Signed-off-by: Naushir Patuck --- include/ipa/raspberrypi.h | 1 + src/ipa/raspberrypi/cam_helper.cpp | 38 ++++++++++++++- src/ipa/raspberrypi/cam_helper.hpp | 15 +++++- src/ipa/raspberrypi/cam_helper_imx219.cpp | 11 ++++- src/ipa/raspberrypi/cam_helper_imx477.cpp | 11 ++++- src/ipa/raspberrypi/cam_helper_ov5647.cpp | 11 ++++- src/ipa/raspberrypi/raspberrypi.cpp | 48 ++++++++++++++++--- .../pipeline/raspberrypi/raspberrypi.cpp | 2 + 8 files changed, 124 insertions(+), 13 deletions(-) diff --git a/include/ipa/raspberrypi.h b/include/ipa/raspberrypi.h index c109469e..74954db2 100644 --- a/include/ipa/raspberrypi.h +++ b/include/ipa/raspberrypi.h @@ -51,6 +51,7 @@ static const ControlInfoMap RPiControls = { { &controls::Brightness, ControlInfo(-1.0f, 1.0f) }, { &controls::Contrast, ControlInfo(0.0f, 32.0f) }, { &controls::Saturation, ControlInfo(0.0f, 32.0f) }, + { &controls::FrameDurations, ControlInfo(1.0e3f, 1.0e9f) } }; } /* namespace libcamera */ diff --git a/src/ipa/raspberrypi/cam_helper.cpp b/src/ipa/raspberrypi/cam_helper.cpp index 7f05d2c6..06732241 100644 --- a/src/ipa/raspberrypi/cam_helper.cpp +++ b/src/ipa/raspberrypi/cam_helper.cpp @@ -34,8 +34,10 @@ CamHelper *CamHelper::Create(std::string const &cam_name) return nullptr; } -CamHelper::CamHelper(MdParser *parser) - : parser_(parser), initialized_(false) +CamHelper::CamHelper(MdParser *parser, unsigned int maxFrameLength, + unsigned int frameIntegrationDiff) + : parser_(parser), initialized_(false), maxFrameLength_(maxFrameLength), + frameIntegrationDiff_(frameIntegrationDiff) { } @@ -56,6 +58,38 @@ double CamHelper::Exposure(uint32_t exposure_lines) const return exposure_lines * mode_.line_length / 1000.0; } +uint32_t CamHelper::GetVBlanking(double &exposure, double minFrameDuration, + double maxFrameDuration) const +{ + uint32_t frameLengthMin, frameLengthMax, vblank; + uint32_t exposureLines = ExposureLines(exposure); + + assert(initialized_); + /* + * Clip frame length by the frame duration range and the maximum allowable + * value in the sensor, given by maxFrameLength_. + */ + frameLengthMin = std::min(1e3 * minFrameDuration / mode_.line_length, + maxFrameLength_); + frameLengthMax = std::min(1e3 * maxFrameDuration / mode_.line_length, + maxFrameLength_); + /* + * Limit the exposure to the maximum frame duration requested, and + * re-calculate if it has been clipped. + */ + exposureLines = std::min(frameLengthMax - frameIntegrationDiff_, + exposureLines); + exposure = Exposure(exposureLines); + + /* Limit the vblank to the range allowed by the frame length limits. */ + vblank = std::max(mode_.height, exposureLines) - + mode_.height + frameIntegrationDiff_; + vblank = std::max(vblank, frameLengthMin - mode_.height); + vblank = std::min(vblank, frameLengthMax - mode_.height); + + return vblank; +} + void CamHelper::SetCameraMode(const CameraMode &mode) { mode_ = mode; diff --git a/src/ipa/raspberrypi/cam_helper.hpp b/src/ipa/raspberrypi/cam_helper.hpp index 0c8aa29a..fdcdbddb 100644 --- a/src/ipa/raspberrypi/cam_helper.hpp +++ b/src/ipa/raspberrypi/cam_helper.hpp @@ -68,12 +68,15 @@ class CamHelper { public: static CamHelper *Create(std::string const &cam_name); - CamHelper(MdParser *parser); + CamHelper(MdParser *parser, unsigned int maxFrameLength, + unsigned int frameIntegrationDiff); virtual ~CamHelper(); void SetCameraMode(const CameraMode &mode); MdParser &Parser() const { return *parser_; } uint32_t ExposureLines(double exposure_us) const; double Exposure(uint32_t exposure_lines) const; // in us + virtual uint32_t GetVBlanking(double &exposure_us, double minFrameDuration, + double maxFrameDuration) const; virtual uint32_t GainCode(double gain) const = 0; virtual double Gain(uint32_t gain_code) const = 0; virtual void GetDelays(int &exposure_delay, int &gain_delay) const; @@ -83,10 +86,20 @@ public: virtual unsigned int MistrustFramesStartup() const; virtual unsigned int MistrustFramesModeSwitch() const; virtual CamTransform GetOrientation() const; + protected: MdParser *parser_; CameraMode mode_; + +private: bool initialized_; + /* Largest possible frame length, in units of lines. */ + unsigned int maxFrameLength_; + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + unsigned int frameIntegrationDiff_; }; // This is for registering camera helpers with the system, so that the diff --git a/src/ipa/raspberrypi/cam_helper_imx219.cpp b/src/ipa/raspberrypi/cam_helper_imx219.cpp index 35c6597c..fd95a31a 100644 --- a/src/ipa/raspberrypi/cam_helper_imx219.cpp +++ b/src/ipa/raspberrypi/cam_helper_imx219.cpp @@ -50,13 +50,22 @@ public: unsigned int MistrustFramesModeSwitch() const override; bool SensorEmbeddedDataPresent() const override; CamTransform GetOrientation() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; + /* Largest possible frame length, in units of lines. */ + static constexpr int maxFrameLength = 0xffff; }; CamHelperImx219::CamHelperImx219() #if ENABLE_EMBEDDED_DATA : CamHelper(new MdParserImx219()) #else - : CamHelper(new MdParserRPi()) + : CamHelper(new MdParserRPi(), maxFrameLength, frameIntegrationDiff) #endif { } diff --git a/src/ipa/raspberrypi/cam_helper_imx477.cpp b/src/ipa/raspberrypi/cam_helper_imx477.cpp index 69544456..4a1cab76 100644 --- a/src/ipa/raspberrypi/cam_helper_imx477.cpp +++ b/src/ipa/raspberrypi/cam_helper_imx477.cpp @@ -39,10 +39,19 @@ public: double Gain(uint32_t gain_code) const override; bool SensorEmbeddedDataPresent() const override; CamTransform GetOrientation() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 22; + /* Largest possible frame length, in units of lines. */ + static constexpr int maxFrameLength = 0xffdc; }; CamHelperImx477::CamHelperImx477() - : CamHelper(new MdParserImx477()) + : CamHelper(new MdParserImx477(), maxFrameLength, frameIntegrationDiff) { } diff --git a/src/ipa/raspberrypi/cam_helper_ov5647.cpp b/src/ipa/raspberrypi/cam_helper_ov5647.cpp index 3dbcb164..d814fa90 100644 --- a/src/ipa/raspberrypi/cam_helper_ov5647.cpp +++ b/src/ipa/raspberrypi/cam_helper_ov5647.cpp @@ -22,6 +22,15 @@ public: unsigned int HideFramesModeSwitch() const override; unsigned int MistrustFramesStartup() const override; unsigned int MistrustFramesModeSwitch() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; + /* Largest possible frame length, in units of lines. */ + static constexpr int maxFrameLength = 0xffff; }; /* @@ -30,7 +39,7 @@ public: */ CamHelperOv5647::CamHelperOv5647() - : CamHelper(new MdParserRPi()) + : CamHelper(new MdParserRPi(), maxFrameLength, frameIntegrationDiff) { } diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp index 3bcc0815..1af5e74a 100644 --- a/src/ipa/raspberrypi/raspberrypi.cpp +++ b/src/ipa/raspberrypi/raspberrypi.cpp @@ -53,6 +53,8 @@ namespace libcamera { /* Configure the sensor with these values initially. */ #define DEFAULT_ANALOGUE_GAIN 1.0 #define DEFAULT_EXPOSURE_TIME 20000 +#define DEFAULT_MIN_FRAME_DURATION (1e6 / 30.0) +#define DEFAULT_MAX_FRAME_DURATION (1e6 / 0.01) LOG_DEFINE_CATEGORY(IPARPI) @@ -136,6 +138,9 @@ private: /* LS table allocation passed in from the pipeline handler. */ uint32_t lsTableHandle_; void *lsTable_; + + /* Frame duration (1/fps) given in microseconds. */ + double minFrameDuration_, maxFrameDuration_; }; int IPARPi::init(const IPASettings &settings) @@ -252,13 +257,20 @@ void IPARPi::configure(const CameraSensorInfo &sensorInfo, controller_.Initialise(); controllerInit_ = true; - /* Calculate initial values for gain and exposure. */ + /* Calculate initial values for gain, vblank, and exposure */ + minFrameDuration_ = DEFAULT_MIN_FRAME_DURATION; + maxFrameDuration_ = DEFAULT_MAX_FRAME_DURATION; + + double exposure = DEFAULT_EXPOSURE_TIME; + int32_t vblank = helper_->GetVBlanking(exposure, minFrameDuration_, + maxFrameDuration_); + int32_t exposure_lines = helper_->ExposureLines(exposure); int32_t gain_code = helper_->GainCode(DEFAULT_ANALOGUE_GAIN); - int32_t exposure_lines = helper_->ExposureLines(DEFAULT_EXPOSURE_TIME); ControlList ctrls(unicam_ctrls_); - ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain_code); + ctrls.set(V4L2_CID_VBLANK, vblank); ctrls.set(V4L2_CID_EXPOSURE, exposure_lines); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain_code); IPAOperationData op; op.operation = RPI_IPA_ACTION_V4L2_SET_STAGGERED; @@ -630,6 +642,17 @@ void IPARPi::queueRequest(const ControlList &controls) break; } + case controls::FRAME_DURATIONS: { + auto frameDurations = ctrl.second.get>(); + + /* This will be applied once AGC recalculations occur. */ + minFrameDuration_ = frameDurations[0]; + maxFrameDuration_ = frameDurations[1]; + libcameraMetadata_.set(controls::FrameDurations, + { frameDurations[0], frameDurations[1] }); + break; + } + default: LOG(IPARPI, Warning) << "Ctrl " << controls::controls.at(ctrl.first)->name() @@ -795,7 +818,12 @@ void IPARPi::applyAGC(const struct AgcStatus *agcStatus) op.operation = RPI_IPA_ACTION_V4L2_SET_STAGGERED; int32_t gain_code = helper_->GainCode(agcStatus->analogue_gain); - int32_t exposure_lines = helper_->ExposureLines(agcStatus->shutter_time); + + /* GetVBlanking might clip exposure time to the fps limits. */ + double exposure = agcStatus->shutter_time; + int32_t vblanking = helper_->GetVBlanking(exposure, minFrameDuration_, + maxFrameDuration_); + int32_t exposure_lines = helper_->ExposureLines(exposure); if (unicam_ctrls_.find(V4L2_CID_ANALOGUE_GAIN) == unicam_ctrls_.end()) { LOG(IPARPI, Error) << "Can't find analogue gain control"; @@ -807,14 +835,20 @@ void IPARPi::applyAGC(const struct AgcStatus *agcStatus) return; } - LOG(IPARPI, Debug) << "Applying AGC Exposure: " << agcStatus->shutter_time - << " (Shutter lines: " << exposure_lines << ") Gain: " + LOG(IPARPI, Debug) << "Applying AGC Exposure: " << exposure + << " (Shutter lines: " << exposure_lines << ", AGC requested " + << agcStatus->shutter_time << ") Gain: " << agcStatus->analogue_gain << " (Gain Code: " << gain_code << ")"; ControlList ctrls(unicam_ctrls_); - ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain_code); + /* + * VBLANK must be set before EXPOSURE as the former will adjust the + * limits of the latter control. + */ + ctrls.set(V4L2_CID_VBLANK, vblanking); ctrls.set(V4L2_CID_EXPOSURE, exposure_lines); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain_code); op.controls.push_back(ctrls); queueFrameAction.emit(0, op); } diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp index 21a1d7f7..948290e2 100644 --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp @@ -1159,7 +1159,9 @@ void RPiCameraData::queueFrameAction(unsigned int frame, const IPAOperationData if (!staggeredCtrl_) { staggeredCtrl_.init(unicam_[Unicam::Image].dev(), { { V4L2_CID_ANALOGUE_GAIN, action.data[0] }, + { V4L2_CID_VBLANK, action.data[1] }, { V4L2_CID_EXPOSURE, action.data[1] } }); + sensorMetadata_ = action.data[2]; }