From patchwork Thu Oct 3 17:49:37 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Niklas_S=C3=B6derlund?= X-Patchwork-Id: 2089 Return-Path: Received: from bin-mail-out-06.binero.net (bin-mail-out-06.binero.net [195.74.38.229]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D20176165B for ; Thu, 3 Oct 2019 19:50:40 +0200 (CEST) X-Halon-ID: 33267621-e606-11e9-837a-0050569116f7 Authorized-sender: niklas@soderlund.pp.se Received: from bismarck.berto.se (unknown [84.172.88.101]) by bin-vsp-out-03.atm.binero.net (Halon) with ESMTPA id 33267621-e606-11e9-837a-0050569116f7; Thu, 03 Oct 2019 19:49:51 +0200 (CEST) From: =?utf-8?q?Niklas_S=C3=B6derlund?= To: libcamera-devel@lists.libcamera.org Date: Thu, 3 Oct 2019 19:49:37 +0200 Message-Id: <20191003174941.1296988-8-niklas.soderlund@ragnatech.se> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> References: <20191003174941.1296988-1-niklas.soderlund@ragnatech.se> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 07/11] libcamera: timeline: Add a basic timeline implementation X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Oct 2019 17:50:41 -0000 The timeline is a helper for pipeline handlers to ease interacting with an IPA. The idea is that the pipeline handler runs a timeline which schedules and executes actions on the hardware. The pipeline listens to signals from the IPA which it translates into actions which are schedule on the timeline. Signed-off-by: Niklas Söderlund --- src/libcamera/include/meson.build | 1 + src/libcamera/include/timeline.h | 71 ++++++++ src/libcamera/meson.build | 1 + src/libcamera/timeline.cpp | 267 ++++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 src/libcamera/include/timeline.h create mode 100644 src/libcamera/timeline.cpp diff --git a/src/libcamera/include/meson.build b/src/libcamera/include/meson.build index 2c74d29bd925ede2..238ddc2c2cf96a5d 100644 --- a/src/libcamera/include/meson.build +++ b/src/libcamera/include/meson.build @@ -18,6 +18,7 @@ libcamera_headers = files([ 'pipeline_handler.h', 'process.h', 'thread.h', + 'timeline.h', 'utils.h', 'v4l2_controls.h', 'v4l2_device.h', diff --git a/src/libcamera/include/timeline.h b/src/libcamera/include/timeline.h new file mode 100644 index 0000000000000000..1ec28a0d9134d35c --- /dev/null +++ b/src/libcamera/include/timeline.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.h - Timeline for per-frame controls + */ +#ifndef __LIBCAMERA_TIMELINE_H__ +#define __LIBCAMERA_TIMELINE_H__ + +#include +#include + +#include + +#include "utils.h" + +namespace libcamera { + +class FrameAction +{ +public: + FrameAction(unsigned int type, unsigned int frame) + : type_(type), frame_(frame) {} + + virtual ~FrameAction() {} + + unsigned int type() const { return type_; } + unsigned int frame() const { return frame_; } + + virtual void run() = 0; + +private: + unsigned int type_; + unsigned int frame_; +}; + +class Timeline +{ +public: + Timeline(); + virtual ~Timeline() {} + + virtual void reset(); + virtual void scheduleAction(FrameAction *action); + virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time); + + utils::duration frameInterval() const { return frameInterval_; } + +protected: + int frameOffset(unsigned int type) const; + utils::duration timeOffset(unsigned int type) const; + + void setRawDelay(unsigned int type, int frame, utils::duration time); + +private: + static constexpr unsigned int HistoryDepth = 10; + + void timeout(Timer *timer); + void updateDeadline(const utils::time_point &deadline); + + std::list> history_; + std::multimap actions_; + std::map> delays_; + utils::duration frameInterval_; + + Timer timer_; +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_TIMELINE_H__ */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 3a3b388a6a344961..195368e77e5e6f01 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -30,6 +30,7 @@ libcamera_sources = files([ 'signal.cpp', 'stream.cpp', 'thread.cpp', + 'timeline.cpp', 'timer.cpp', 'utils.cpp', 'v4l2_controls.cpp', diff --git a/src/libcamera/timeline.cpp b/src/libcamera/timeline.cpp new file mode 100644 index 0000000000000000..042ead41d2ff54d5 --- /dev/null +++ b/src/libcamera/timeline.cpp @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Google Inc. + * + * timeline.cpp - Timeline for per-frame controls + */ + +#include "timeline.h" + +#include "log.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Timeline) + +/** + * \class FrameAction + * \brief Action which can be schedule within a frame's lifetime + * + * A frame action is an event which takes place at some point during a frame's + * lifetime inside libcamera. Each action has two primal attributes; type and + * frame number. + * + * The type is a numerical ID which identifies the action within the pipeline + * handler. The type is pipeline specific and has no meaning outside the + * specific implementation. The frame number is the frame the action applies to. + */ + +/** + * \fn FrameAction::FrameAction + * \brief Create a frame action + * \param[in] type Action type + * \param[in] frame Frame number + */ + +/** + * \fn FrameAction::type() + * \brief Retrieve the type of action + * \return Action type + */ + +/** + * \fn FrameAction::frame() + * \brief Retrieve the frame number + * \return Frame number + */ + +/** + * \fn FrameAction::run() + * \brief The action to perform when the action is triggered + * + * Pipeline handlers shall subclass the FrameAction object and implement run() + * functions which describes the actions they wish to happen when the act is + * schedule using the Timeline. + * + * \sa Timeline + */ + +/** + * \class Timeline + * \brief Scheduler and executor of FrameAction's + * + * On the timeline the pipeline can schedule FrameAction and expect them to be + * executed at the correct time. The correct time to execute them is a sum + * of which frame the action should apply to and a list of timing delays for + * each action the pipeline handler describes. + * + * The timing delays can either be set by the pipeline handler or by the IPA. + * The exact implementation of how the timing delays are setup is pipeline + * specific. It is expected that the IPA will update the timing delays as it + * make changes to how the pipeline operates in different situations. + * + * The pipeline handler is responsible for feeding the timeline as accurate as + * possible information of the exposures it observes. + */ + +Timeline::Timeline() + : frameInterval_(std::chrono::milliseconds(1)) +{ + timer_.timeout.connect(this, &Timeline::timeout); +} + +/** + * \brief Reset the timeline + */ +void Timeline::reset() +{ + timer_.stop(); + + actions_.clear(); + history_.clear(); +} + +/** + * \brief Add a action to the timeline + * \param[in] action FrameAction to add + * + * Schedule an action at a specific time taking the action and timing delays + * into account. If a action is schedule too late execute it immediately and + * try to recover. + */ +void Timeline::scheduleAction(FrameAction *action) +{ + unsigned int lastFrame; + utils::time_point lastTime; + + if (history_.empty()) { + /* + * This only happens for the first number of frames, up to + * pipeline depth. + */ + lastFrame = 0; + lastTime = std::chrono::steady_clock::now(); + } else { + lastFrame = history_.back().first; + lastTime = history_.back().second; + } + + /* + * Calculate action trigger time by first figuring the out the start of + * exposure (SOE) of the frame the action corresponds to and then adding + * the frame and timing offsets. + */ + int frame = action->frame() + frameOffset(action->type()) - lastFrame; + utils::time_point deadline = lastTime + frame * frameInterval_ + + timeOffset(action->type()); + + utils::time_point now = std::chrono::steady_clock::now(); + if (deadline < now) { + LOG(Timeline, Warning) + << "Action scheduled too late " + << utils::time_point_to_string(deadline) + << ", run now " << utils::time_point_to_string(now); + action->run(); + } else { + actions_.insert({ deadline, action }); + updateDeadline(deadline); + } +} + +/** + * \brief Inform timeline of a new exposure + * \param[in] frame Frame number of the exposure + * \param[in] time The best approximation of when the exposure started + */ +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time) +{ + history_.push_back(std::make_pair(frame, time)); + + if (history_.size() <= HistoryDepth / 2) + return; + + while (history_.size() > HistoryDepth) + history_.pop_front(); + + /* Update esitmated time between two start of exposures. */ + utils::duration sumExposures = std::chrono::duration_values::zero(); + unsigned int numExposures = 0; + + utils::time_point lastTime; + for (auto it = history_.begin(); it != history_.end(); it++) { + if (it != history_.begin()) { + sumExposures += it->second - lastTime; + numExposures++; + } + + lastTime = it->second; + } + + frameInterval_ = sumExposures; + if (numExposures) + frameInterval_ /= numExposures; +} + +/** + * \fn Timeline::frameInterval() + * \brief Retrieve the best estimate of the frame interval + * \return Frame interval estimate + */ + +/** + * \brief Retrieve the frame offset for an action type + * \param[in] type Action type + * \return Frame offset for the action type + */ +int Timeline::frameOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No frame offset set for action type " << type; + return 0; + } + + return it->second.first; +} + +/** + * \brief Retrieve the time offset for an action type + * \param[in] type Action type + * \return Time offset for the action type + */ +utils::duration Timeline::timeOffset(unsigned int type) const +{ + const auto it = delays_.find(type); + if (it == delays_.end()) { + LOG(Timeline, Error) + << "No time offset set for action type " << type; + return utils::duration::zero(); + } + + return it->second.second; +} + +/** + * \brief Set the frame and time offset delays for an action type + * \param[in] type Action type + * \param[in] frame Frame offset + * \param[in] time Time offset + */ +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time) +{ + delays_[type] = std::make_pair(frame, time); +} + +void Timeline::updateDeadline(const utils::time_point &deadline) +{ + if (timer_.isRunning() && deadline >= timer_.deadline()) + return; + + utils::time_point now = std::chrono::steady_clock::now(); + utils::duration duration = deadline - now; + + /* + * Translate nanoseconds resolution the timeline to milliseconds using + * a ceiling approach to not trigger an action before it's scheduled. + * + * \todo: Should the Timer operate using nanoseconds? + */ + unsigned long nsecs = std::chrono::duration_cast(duration).count(); + unsigned int msecs = nsecs / 1000000 + (nsecs % 1000000 ? 1 : 0); + + timer_.stop(); + timer_.start(msecs); +} + +void Timeline::timeout(Timer *timer) +{ + for (auto it = actions_.begin(); it != actions_.end();) { + utils::time_point now = std::chrono::steady_clock::now(); + + utils::time_point sched = it->first; + FrameAction *action = it->second; + + if (sched >= now) { + updateDeadline(sched); + break; + } + + action->run(); + + it = actions_.erase(it); + delete action; + } +} + +} /* namespace libcamera */