Timestamps improvements

Moves from QElapsedTimer to chrono::steady_clock.
Uses frame timestamps provided by the frame if available, compensating
for drifts w.r.t. EyeRec time.

EyeRecToo/EyeRecToo.pro

@@ -120,7 +120,8 @@ HEADERS  += \
     $${TOP}/src/post-processing/TSVReader.h \
     $${TOP}/src/post-processing/VideoSource.h \
     $${TOP}/src/gaze-estimation/GazeEstimate.h \
-    $${TOP}/src/gaze-estimation/GazeEstimate.h
+    $${TOP}/src/gaze-estimation/GazeEstimate.h \
+    $${TOP}/src/MonotonicClock.h
 
 FORMS    += \
     $${TOP}/src/MainWindow.ui \

EyeRecToo/src/AudioRecorder.cpp

@@ -5,13 +5,18 @@
 
 AudioRecorder::AudioRecorder(QObject* parent)
     : QObject(parent)
+    , recorder(nullptr)
+    , recording(false)
 {
 }
 
 AudioRecorder::~AudioRecorder()
 {
-    if (recorder && recording)
-        recorder->stop();
+    if (recorder) {
+        if (recording)
+            recorder->stop();
+        recorder->deleteLater();
+    }
 }
 
 void AudioRecorder::startRecording()

EyeRecToo/src/AudioRecorder.h

@@ -20,8 +20,8 @@ public slots:
     void stopRecording();
 
 private:
-    QAudioRecorder* recorder = nullptr;
-    bool recording = false;
+    QAudioRecorder* recorder;
+    bool recording;
     Timestamp start;
 };
 

EyeRecToo/src/EyeImageProcessor.cpp

@@ -60,7 +60,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
         return;
 
     QMutexLocker locker(&cfgMutex);
-    Timestamp processingStartNs = gTimer.nsecsElapsed();
+    Timestamp processingStart = gTimer.elapsed();
 
     data.timestamp = timestamp;
 
@@ -136,7 +136,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
     data.modelData = EyeModelData();
 
     data.cameraCalibration = cameraCalibration;
-    data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs);
+    data.processingTimestamp = gTimer.elapsed() - processingStart;
 
     emit newData(data);
 }

EyeRecToo/src/FieldImageProcessor.cpp

@@ -46,7 +46,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
         return;
 
     QMutexLocker locker(&cfgMutex);
-    Timestamp processingStartNs = gTimer.nsecsElapsed();
+    Timestamp processingStart = gTimer.elapsed();
 
     data.timestamp = timestamp;
 
@@ -135,7 +135,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
 
     data.gazeEstimate.valid = false;
     data.cameraCalibration = cameraCalibration;
-    data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs);
+    data.processingTimestamp = gTimer.elapsed() - processingStart;
 
     emit newData(data);
 }

EyeRecToo/src/FrameGrabber.cpp

@@ -11,7 +11,6 @@ FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
     , code(code)
     , yuvBuffer(nullptr)
     , yuvBufferSize(0)
-    , timestampOffset(0)
     , timeoutMs(2e3)
 {
 #ifdef TURBOJPEG
@@ -74,6 +73,37 @@ QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVid
     //<< QVideoFrame::Format_AdobeDng;
 }
 
+Timestamp FrameGrabber::getTimestamp(const QVideoFrame& frame)
+{
+    /* We have three options for timestamps:
+     * 1 EyeRec software timestamp (t)
+     * 2 Frame presentation timestamp (pts)
+     * 3 Frame software timestamp (fts) -- at least for the uvcengine
+     *
+     * However, we can't guarante the source for pts and fts, so we need to keep
+     * track of drift
+     */
+    auto ts = gTimer.elapsed();
+
+    // 1
+    auto selected = ts;
+
+    if (frame.startTime() > 0 && frame.endTime() > 0) {
+        // 2
+        const auto pts = 1e-3 * 0.5 * (frame.endTime() + frame.startTime());
+        selected = drift.correct(ts, pts);
+    } else {
+        // 3
+        const auto& metaNow = frame.metaData("steady_clock::now");
+        if (metaNow.isValid()) {
+            const auto fts = gTimer.elapsed(qvariant_cast<SteadyTimePoint>(metaNow));
+            selected = drift.correct(ts, fts);
+        }
+    }
+
+    return selected;
+}
+
 bool FrameGrabber::present(const QVideoFrame& frame)
 {
     /*
@@ -85,22 +115,10 @@ bool FrameGrabber::present(const QVideoFrame& frame)
      * we must copy the data
      *
      */
-    Timestamp t = ns2ms(gTimer.nsecsElapsed());
-
-    /* TODO: Disabled for the time being; the number of samples here should be
-     * better adjusted otherwise we risk getting the offset wrong
-    const QString tMetaStr = "timestamp";
-    if (frame.metaData(tMetaStr).isValid()) {
-        Timestamp ft = qvariant_cast<Timestamp>(frame.metaData(tMetaStr));
-        if (timestampOffsetEstimators.size() < 30) {
-            // Use median of first N frames to estimate the offset
-            timestampOffsetEstimators.emplace_back(t - ft);
-            std::sort(timestampOffsetEstimators.begin(), timestampOffsetEstimators.end());
-            timestampOffset = timestampOffsetEstimators[0.5 * (timestampOffsetEstimators.size() - 1)];
-        }
-        t = ft + timestampOffset;
-    }
-    */
+    using namespace std::chrono;
+
+    // Get SW timestamp asap
+    auto t = getTimestamp(frame);
 
     if (!frame.isValid())
         return false;

EyeRecToo/src/FrameGrabber.h

 #ifndef FRAMEGRABBER_H
 #define FRAMEGRABBER_H
 
+#include <chrono>
+
 #include <QAbstractVideoSurface>
 #include <QDebug>
-#include <QElapsedTimer>
 #include <QThread>
 #include <QTimer>
 #include <QVideoSurfaceFormat>
@@ -16,6 +17,38 @@
 
 #include "utils.h"
 
+Q_DECLARE_METATYPE(SteadyTimePoint)
+
+class Drift {
+public:
+    Drift(const Timestamp tolerance = 10)
+        : tolerance(tolerance)
+    {
+    }
+    void setTolerance(const Timestamp newTolerance) { tolerance = newTolerance; }
+    Timestamp correct(const Timestamp ref, const Timestamp other)
+    {
+        using std::max;
+
+        auto dt = other - prev;
+        auto drift = ref - (cur + dt);
+
+        // if drift is too large, we discard dt and update cur monotonically
+        if (abs(drift) > tolerance)
+            cur = max(ref, cur);
+        else
+            cur += dt;
+
+        prev = other;
+        return cur;
+    }
+
+private:
+    Timestamp tolerance;
+    Timestamp prev;
+    Timestamp cur;
+};
+
 class FrameGrabber : public QAbstractVideoSurface {
     Q_OBJECT
 public:
@@ -34,8 +67,6 @@ public slots:
 private:
     QTimer* watchdog;
     int timeoutMs;
-    Timestamp timestampOffset;
-    std::vector<Timestamp> timestampOffsetEstimators;
 #ifdef TURBOJPEG
     tjhandle tjh;
 #endif
@@ -48,6 +79,9 @@ private:
     bool yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame);
 
     unsigned int pmIdx;
+
+    Drift drift;
+    Timestamp getTimestamp(const QVideoFrame& frame);
 };
 
 #endif // FRAMEGRABBER_H

EyeRecToo/src/GazeEstimation.cpp

@@ -756,4 +756,5 @@ void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>&
 
 void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples)
 {
+    (void)tuples;
 }

EyeRecToo/src/PerformanceMonitor.cpp

@@ -39,7 +39,7 @@ unsigned int PerformanceMonitor::enrol(const QString& id, const QString& stage)
     return idx;
 }
 
-extern QElapsedTimer gTimer;
+extern ReferenceClock gTimer;
 extern bool gPostProcessing;
 bool PerformanceMonitor::shouldDrop(const unsigned int& idx, const Timestamp& timestamp, const int& maxDelay)
 {

EyeRecToo/src/globals.cpp

@@ -7,7 +7,7 @@ bool gPostProcessing = false;
 LogWidget* gLogWidget = nullptr;
 
 // Initialized on startup
-QElapsedTimer gTimer;
+ReferenceClock gTimer;
 QString gExeDir;
 QString gCfgDir;
 PerformanceMonitor gPerformanceMonitor;

EyeRecToo/src/globals.h

 #ifndef GLOBALS_H
 #define GLOBALS_H
 
-#include <QElapsedTimer>
 #include <QMutex>
 
 #include "LogWidget.h"
 #include "PerformanceMonitor.h"
 
+#include "MonotonicClock.h"
+
 // Time reference
-extern QElapsedTimer gTimer;
+extern ReferenceClock gTimer;
 
 // Executable directory
 extern QString gExeDir;

EyeRecToo/src/main.cpp

@@ -36,12 +36,8 @@ int main(int argc, char* argv[])
 #endif
 
     qInstallMessageHandler(logMessages);
-    gTimer.start();
     logInitBanner();
 
-    if (!gTimer.isMonotonic()) // TODO: give the user the option to continue with an unreliable clock
-        qFatal("Non monotonic reference clock.");
-
     makeFolders();
 
     /*

EyeRecToo/src/post-processing/VideoSource.h

@@ -3,6 +3,7 @@
 
 #include <memory>
 
+#include <QElapsedTimer>
 #include <QFileInfo>
 #include <QObject>
 #include <QThread>

EyeRecToo/src/utils.cpp

 #include "utils.h"
 
+#include <QElapsedTimer>
+
 const char Token::Delimiter = '\t';
 const char Token::Newline = '\n';
 const char Token::JournalEntry = 'J';
@@ -52,7 +54,7 @@ void logExitBanner()
     logStream.flush();
 }
 
-extern QElapsedTimer gTimer;
+extern ReferenceClock gTimer;
 extern LogWidget* gLogWidget;
 void logMessages(QtMsgType type, const QMessageLogContext& context, const QString& msg)
 {

EyeRecToo/src/utils.h

@@ -8,7 +8,6 @@
 #include <QCameraViewfinderSettings>
 #include <QDateTime>
 #include <QDebug>
-#include <QElapsedTimer>
 #include <QFile>
 #include <QMutex>
 #include <QTextStream>
@@ -22,7 +21,8 @@ Q_DECLARE_METATYPE(QCameraInfo)
 
 #include "LogWidget.h"
 
-typedef qint64 Timestamp;
+#include "MonotonicClock.h"
+
 extern const Timestamp maxTimestamp;
 Q_DECLARE_METATYPE(Timestamp)