Commit bd209a25 authored by Thiago Santini's avatar Thiago Santini

Timestamps improvements

Moves from QElapsedTimer to chrono::steady_clock.
Uses frame timestamps provided by the frame if available, compensating
for drifts w.r.t. EyeRec time.
parent 3b896cce
...@@ -120,7 +120,8 @@ HEADERS += \ ...@@ -120,7 +120,8 @@ HEADERS += \
$${TOP}/src/post-processing/TSVReader.h \ $${TOP}/src/post-processing/TSVReader.h \
$${TOP}/src/post-processing/VideoSource.h \ $${TOP}/src/post-processing/VideoSource.h \
$${TOP}/src/gaze-estimation/GazeEstimate.h \ $${TOP}/src/gaze-estimation/GazeEstimate.h \
$${TOP}/src/gaze-estimation/GazeEstimate.h $${TOP}/src/gaze-estimation/GazeEstimate.h \
$${TOP}/src/MonotonicClock.h
FORMS += \ FORMS += \
$${TOP}/src/MainWindow.ui \ $${TOP}/src/MainWindow.ui \
......
...@@ -5,13 +5,18 @@ ...@@ -5,13 +5,18 @@
AudioRecorder::AudioRecorder(QObject* parent) AudioRecorder::AudioRecorder(QObject* parent)
: QObject(parent) : QObject(parent)
, recorder(nullptr)
, recording(false)
{ {
} }
AudioRecorder::~AudioRecorder() AudioRecorder::~AudioRecorder()
{ {
if (recorder && recording) if (recorder) {
recorder->stop(); if (recording)
recorder->stop();
recorder->deleteLater();
}
} }
void AudioRecorder::startRecording() void AudioRecorder::startRecording()
......
...@@ -20,8 +20,8 @@ public slots: ...@@ -20,8 +20,8 @@ public slots:
void stopRecording(); void stopRecording();
private: private:
QAudioRecorder* recorder = nullptr; QAudioRecorder* recorder;
bool recording = false; bool recording;
Timestamp start; Timestamp start;
}; };
......
...@@ -60,7 +60,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame) ...@@ -60,7 +60,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
return; return;
QMutexLocker locker(&cfgMutex); QMutexLocker locker(&cfgMutex);
Timestamp processingStartNs = gTimer.nsecsElapsed(); Timestamp processingStart = gTimer.elapsed();
data.timestamp = timestamp; data.timestamp = timestamp;
...@@ -136,7 +136,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame) ...@@ -136,7 +136,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
data.modelData = EyeModelData(); data.modelData = EyeModelData();
data.cameraCalibration = cameraCalibration; data.cameraCalibration = cameraCalibration;
data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs); data.processingTimestamp = gTimer.elapsed() - processingStart;
emit newData(data); emit newData(data);
} }
......
...@@ -46,7 +46,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame) ...@@ -46,7 +46,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
return; return;
QMutexLocker locker(&cfgMutex); QMutexLocker locker(&cfgMutex);
Timestamp processingStartNs = gTimer.nsecsElapsed(); Timestamp processingStart = gTimer.elapsed();
data.timestamp = timestamp; data.timestamp = timestamp;
...@@ -135,7 +135,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame) ...@@ -135,7 +135,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
data.gazeEstimate.valid = false; data.gazeEstimate.valid = false;
data.cameraCalibration = cameraCalibration; data.cameraCalibration = cameraCalibration;
data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs); data.processingTimestamp = gTimer.elapsed() - processingStart;
emit newData(data); emit newData(data);
} }
......
...@@ -11,7 +11,6 @@ FrameGrabber::FrameGrabber(QString id, int code, QObject* parent) ...@@ -11,7 +11,6 @@ FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
, code(code) , code(code)
, yuvBuffer(nullptr) , yuvBuffer(nullptr)
, yuvBufferSize(0) , yuvBufferSize(0)
, timestampOffset(0)
, timeoutMs(2e3) , timeoutMs(2e3)
{ {
#ifdef TURBOJPEG #ifdef TURBOJPEG
...@@ -74,6 +73,37 @@ QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVid ...@@ -74,6 +73,37 @@ QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVid
//<< QVideoFrame::Format_AdobeDng; //<< QVideoFrame::Format_AdobeDng;
} }
Timestamp FrameGrabber::getTimestamp(const QVideoFrame& frame)
{
/* We have three options for timestamps:
* 1 EyeRec software timestamp (t)
* 2 Frame presentation timestamp (pts)
* 3 Frame software timestamp (fts) -- at least for the uvcengine
*
* However, we can't guarante the source for pts and fts, so we need to keep
* track of drift
*/
auto ts = gTimer.elapsed();
// 1
auto selected = ts;
if (frame.startTime() > 0 && frame.endTime() > 0) {
// 2
const auto pts = 1e-3 * 0.5 * (frame.endTime() + frame.startTime());
selected = drift.correct(ts, pts);
} else {
// 3
const auto& metaNow = frame.metaData("steady_clock::now");
if (metaNow.isValid()) {
const auto fts = gTimer.elapsed(qvariant_cast<SteadyTimePoint>(metaNow));
selected = drift.correct(ts, fts);
}
}
return selected;
}
bool FrameGrabber::present(const QVideoFrame& frame) bool FrameGrabber::present(const QVideoFrame& frame)
{ {
/* /*
...@@ -85,22 +115,10 @@ bool FrameGrabber::present(const QVideoFrame& frame) ...@@ -85,22 +115,10 @@ bool FrameGrabber::present(const QVideoFrame& frame)
* we must copy the data * we must copy the data
* *
*/ */
Timestamp t = ns2ms(gTimer.nsecsElapsed()); using namespace std::chrono;
/* TODO: Disabled for the time being; the number of samples here should be // Get SW timestamp asap
* better adjusted otherwise we risk getting the offset wrong auto t = getTimestamp(frame);
const QString tMetaStr = "timestamp";
if (frame.metaData(tMetaStr).isValid()) {
Timestamp ft = qvariant_cast<Timestamp>(frame.metaData(tMetaStr));
if (timestampOffsetEstimators.size() < 30) {
// Use median of first N frames to estimate the offset
timestampOffsetEstimators.emplace_back(t - ft);
std::sort(timestampOffsetEstimators.begin(), timestampOffsetEstimators.end());
timestampOffset = timestampOffsetEstimators[0.5 * (timestampOffsetEstimators.size() - 1)];
}
t = ft + timestampOffset;
}
*/
if (!frame.isValid()) if (!frame.isValid())
return false; return false;
......
#ifndef FRAMEGRABBER_H #ifndef FRAMEGRABBER_H
#define FRAMEGRABBER_H #define FRAMEGRABBER_H
#include <chrono>
#include <QAbstractVideoSurface> #include <QAbstractVideoSurface>
#include <QDebug> #include <QDebug>
#include <QElapsedTimer>
#include <QThread> #include <QThread>
#include <QTimer> #include <QTimer>
#include <QVideoSurfaceFormat> #include <QVideoSurfaceFormat>
...@@ -16,6 +17,38 @@ ...@@ -16,6 +17,38 @@
#include "utils.h" #include "utils.h"
Q_DECLARE_METATYPE(SteadyTimePoint)
class Drift {
public:
Drift(const Timestamp tolerance = 10)
: tolerance(tolerance)
{
}
void setTolerance(const Timestamp newTolerance) { tolerance = newTolerance; }
Timestamp correct(const Timestamp ref, const Timestamp other)
{
using std::max;
auto dt = other - prev;
auto drift = ref - (cur + dt);
// if drift is too large, we discard dt and update cur monotonically
if (abs(drift) > tolerance)
cur = max(ref, cur);
else
cur += dt;
prev = other;
return cur;
}
private:
Timestamp tolerance;
Timestamp prev;
Timestamp cur;
};
class FrameGrabber : public QAbstractVideoSurface { class FrameGrabber : public QAbstractVideoSurface {
Q_OBJECT Q_OBJECT
public: public:
...@@ -34,8 +67,6 @@ public slots: ...@@ -34,8 +67,6 @@ public slots:
private: private:
QTimer* watchdog; QTimer* watchdog;
int timeoutMs; int timeoutMs;
Timestamp timestampOffset;
std::vector<Timestamp> timestampOffsetEstimators;
#ifdef TURBOJPEG #ifdef TURBOJPEG
tjhandle tjh; tjhandle tjh;
#endif #endif
...@@ -48,6 +79,9 @@ private: ...@@ -48,6 +79,9 @@ private:
bool yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame); bool yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame);
unsigned int pmIdx; unsigned int pmIdx;
Drift drift;
Timestamp getTimestamp(const QVideoFrame& frame);
}; };
#endif // FRAMEGRABBER_H #endif // FRAMEGRABBER_H
...@@ -756,4 +756,5 @@ void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>& ...@@ -756,4 +756,5 @@ void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>&
void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples) void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples)
{ {
(void)tuples;
} }
...@@ -39,7 +39,7 @@ unsigned int PerformanceMonitor::enrol(const QString& id, const QString& stage) ...@@ -39,7 +39,7 @@ unsigned int PerformanceMonitor::enrol(const QString& id, const QString& stage)
return idx; return idx;
} }
extern QElapsedTimer gTimer; extern ReferenceClock gTimer;
extern bool gPostProcessing; extern bool gPostProcessing;
bool PerformanceMonitor::shouldDrop(const unsigned int& idx, const Timestamp& timestamp, const int& maxDelay) bool PerformanceMonitor::shouldDrop(const unsigned int& idx, const Timestamp& timestamp, const int& maxDelay)
{ {
......
...@@ -7,7 +7,7 @@ bool gPostProcessing = false; ...@@ -7,7 +7,7 @@ bool gPostProcessing = false;
LogWidget* gLogWidget = nullptr; LogWidget* gLogWidget = nullptr;
// Initialized on startup // Initialized on startup
QElapsedTimer gTimer; ReferenceClock gTimer;
QString gExeDir; QString gExeDir;
QString gCfgDir; QString gCfgDir;
PerformanceMonitor gPerformanceMonitor; PerformanceMonitor gPerformanceMonitor;
......
#ifndef GLOBALS_H #ifndef GLOBALS_H
#define GLOBALS_H #define GLOBALS_H
#include <QElapsedTimer>
#include <QMutex> #include <QMutex>
#include "LogWidget.h" #include "LogWidget.h"
#include "PerformanceMonitor.h" #include "PerformanceMonitor.h"
#include "MonotonicClock.h"
// Time reference // Time reference
extern QElapsedTimer gTimer; extern ReferenceClock gTimer;
// Executable directory // Executable directory
extern QString gExeDir; extern QString gExeDir;
......
...@@ -36,12 +36,8 @@ int main(int argc, char* argv[]) ...@@ -36,12 +36,8 @@ int main(int argc, char* argv[])
#endif #endif
qInstallMessageHandler(logMessages); qInstallMessageHandler(logMessages);
gTimer.start();
logInitBanner(); logInitBanner();
if (!gTimer.isMonotonic()) // TODO: give the user the option to continue with an unreliable clock
qFatal("Non monotonic reference clock.");
makeFolders(); makeFolders();
/* /*
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#include <memory> #include <memory>
#include <QElapsedTimer>
#include <QFileInfo> #include <QFileInfo>
#include <QObject> #include <QObject>
#include <QThread> #include <QThread>
......
#include "utils.h" #include "utils.h"
#include <QElapsedTimer>
const char Token::Delimiter = '\t'; const char Token::Delimiter = '\t';
const char Token::Newline = '\n'; const char Token::Newline = '\n';
const char Token::JournalEntry = 'J'; const char Token::JournalEntry = 'J';
...@@ -52,7 +54,7 @@ void logExitBanner() ...@@ -52,7 +54,7 @@ void logExitBanner()
logStream.flush(); logStream.flush();
} }
extern QElapsedTimer gTimer; extern ReferenceClock gTimer;
extern LogWidget* gLogWidget; extern LogWidget* gLogWidget;
void logMessages(QtMsgType type, const QMessageLogContext& context, const QString& msg) void logMessages(QtMsgType type, const QMessageLogContext& context, const QString& msg)
{ {
......
...@@ -8,7 +8,6 @@ ...@@ -8,7 +8,6 @@
#include <QCameraViewfinderSettings> #include <QCameraViewfinderSettings>
#include <QDateTime> #include <QDateTime>
#include <QDebug> #include <QDebug>
#include <QElapsedTimer>
#include <QFile> #include <QFile>
#include <QMutex> #include <QMutex>
#include <QTextStream> #include <QTextStream>
...@@ -22,7 +21,8 @@ Q_DECLARE_METATYPE(QCameraInfo) ...@@ -22,7 +21,8 @@ Q_DECLARE_METATYPE(QCameraInfo)
#include "LogWidget.h" #include "LogWidget.h"
typedef qint64 Timestamp; #include "MonotonicClock.h"
extern const Timestamp maxTimestamp; extern const Timestamp maxTimestamp;
Q_DECLARE_METATYPE(Timestamp) Q_DECLARE_METATYPE(Timestamp)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment