Commit bd209a25 authored by Thiago Santini's avatar Thiago Santini

Timestamps improvements

Moves from QElapsedTimer to chrono::steady_clock.
Uses frame timestamps provided by the frame if available, compensating
for drifts w.r.t. EyeRec time.
parent 3b896cce
......@@ -120,7 +120,8 @@ HEADERS += \
$${TOP}/src/post-processing/TSVReader.h \
$${TOP}/src/post-processing/VideoSource.h \
$${TOP}/src/gaze-estimation/GazeEstimate.h \
$${TOP}/src/gaze-estimation/GazeEstimate.h
$${TOP}/src/gaze-estimation/GazeEstimate.h \
$${TOP}/src/MonotonicClock.h
FORMS += \
$${TOP}/src/MainWindow.ui \
......
......@@ -5,13 +5,18 @@
AudioRecorder::AudioRecorder(QObject* parent)
: QObject(parent)
, recorder(nullptr)
, recording(false)
{
}
AudioRecorder::~AudioRecorder()
{
if (recorder && recording)
recorder->stop();
if (recorder) {
if (recording)
recorder->stop();
recorder->deleteLater();
}
}
void AudioRecorder::startRecording()
......
......@@ -20,8 +20,8 @@ public slots:
void stopRecording();
private:
QAudioRecorder* recorder = nullptr;
bool recording = false;
QAudioRecorder* recorder;
bool recording;
Timestamp start;
};
......
......@@ -60,7 +60,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
return;
QMutexLocker locker(&cfgMutex);
Timestamp processingStartNs = gTimer.nsecsElapsed();
Timestamp processingStart = gTimer.elapsed();
data.timestamp = timestamp;
......@@ -136,7 +136,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
data.modelData = EyeModelData();
data.cameraCalibration = cameraCalibration;
data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs);
data.processingTimestamp = gTimer.elapsed() - processingStart;
emit newData(data);
}
......
......@@ -46,7 +46,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
return;
QMutexLocker locker(&cfgMutex);
Timestamp processingStartNs = gTimer.nsecsElapsed();
Timestamp processingStart = gTimer.elapsed();
data.timestamp = timestamp;
......@@ -135,7 +135,7 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat& frame)
data.gazeEstimate.valid = false;
data.cameraCalibration = cameraCalibration;
data.processingTimestamp = ns2ms(gTimer.nsecsElapsed() - processingStartNs);
data.processingTimestamp = gTimer.elapsed() - processingStart;
emit newData(data);
}
......
......@@ -11,7 +11,6 @@ FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
, code(code)
, yuvBuffer(nullptr)
, yuvBufferSize(0)
, timestampOffset(0)
, timeoutMs(2e3)
{
#ifdef TURBOJPEG
......@@ -74,6 +73,37 @@ QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVid
//<< QVideoFrame::Format_AdobeDng;
}
Timestamp FrameGrabber::getTimestamp(const QVideoFrame& frame)
{
/* We have three options for timestamps:
* 1 EyeRec software timestamp (t)
* 2 Frame presentation timestamp (pts)
* 3 Frame software timestamp (fts) -- at least for the uvcengine
*
* However, we can't guarante the source for pts and fts, so we need to keep
* track of drift
*/
auto ts = gTimer.elapsed();
// 1
auto selected = ts;
if (frame.startTime() > 0 && frame.endTime() > 0) {
// 2
const auto pts = 1e-3 * 0.5 * (frame.endTime() + frame.startTime());
selected = drift.correct(ts, pts);
} else {
// 3
const auto& metaNow = frame.metaData("steady_clock::now");
if (metaNow.isValid()) {
const auto fts = gTimer.elapsed(qvariant_cast<SteadyTimePoint>(metaNow));
selected = drift.correct(ts, fts);
}
}
return selected;
}
bool FrameGrabber::present(const QVideoFrame& frame)
{
/*
......@@ -85,22 +115,10 @@ bool FrameGrabber::present(const QVideoFrame& frame)
* we must copy the data
*
*/
Timestamp t = ns2ms(gTimer.nsecsElapsed());
/* TODO: Disabled for the time being; the number of samples here should be
* better adjusted otherwise we risk getting the offset wrong
const QString tMetaStr = "timestamp";
if (frame.metaData(tMetaStr).isValid()) {
Timestamp ft = qvariant_cast<Timestamp>(frame.metaData(tMetaStr));
if (timestampOffsetEstimators.size() < 30) {
// Use median of first N frames to estimate the offset
timestampOffsetEstimators.emplace_back(t - ft);
std::sort(timestampOffsetEstimators.begin(), timestampOffsetEstimators.end());
timestampOffset = timestampOffsetEstimators[0.5 * (timestampOffsetEstimators.size() - 1)];
}
t = ft + timestampOffset;
}
*/
using namespace std::chrono;
// Get SW timestamp asap
auto t = getTimestamp(frame);
if (!frame.isValid())
return false;
......
#ifndef FRAMEGRABBER_H
#define FRAMEGRABBER_H
#include <chrono>
#include <QAbstractVideoSurface>
#include <QDebug>
#include <QElapsedTimer>
#include <QThread>
#include <QTimer>
#include <QVideoSurfaceFormat>
......@@ -16,6 +17,38 @@
#include "utils.h"
Q_DECLARE_METATYPE(SteadyTimePoint)
class Drift {
public:
Drift(const Timestamp tolerance = 10)
: tolerance(tolerance)
{
}
void setTolerance(const Timestamp newTolerance) { tolerance = newTolerance; }
Timestamp correct(const Timestamp ref, const Timestamp other)
{
using std::max;
auto dt = other - prev;
auto drift = ref - (cur + dt);
// if drift is too large, we discard dt and update cur monotonically
if (abs(drift) > tolerance)
cur = max(ref, cur);
else
cur += dt;
prev = other;
return cur;
}
private:
Timestamp tolerance;
Timestamp prev;
Timestamp cur;
};
class FrameGrabber : public QAbstractVideoSurface {
Q_OBJECT
public:
......@@ -34,8 +67,6 @@ public slots:
private:
QTimer* watchdog;
int timeoutMs;
Timestamp timestampOffset;
std::vector<Timestamp> timestampOffsetEstimators;
#ifdef TURBOJPEG
tjhandle tjh;
#endif
......@@ -48,6 +79,9 @@ private:
bool yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame);
unsigned int pmIdx;
Drift drift;
Timestamp getTimestamp(const QVideoFrame& frame);
};
#endif // FRAMEGRABBER_H
......@@ -756,4 +756,5 @@ void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>&
void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples)
{
(void)tuples;
}
......@@ -39,7 +39,7 @@ unsigned int PerformanceMonitor::enrol(const QString& id, const QString& stage)
return idx;
}
extern QElapsedTimer gTimer;
extern ReferenceClock gTimer;
extern bool gPostProcessing;
bool PerformanceMonitor::shouldDrop(const unsigned int& idx, const Timestamp& timestamp, const int& maxDelay)
{
......
......@@ -7,7 +7,7 @@ bool gPostProcessing = false;
LogWidget* gLogWidget = nullptr;
// Initialized on startup
QElapsedTimer gTimer;
ReferenceClock gTimer;
QString gExeDir;
QString gCfgDir;
PerformanceMonitor gPerformanceMonitor;
......
#ifndef GLOBALS_H
#define GLOBALS_H
#include <QElapsedTimer>
#include <QMutex>
#include "LogWidget.h"
#include "PerformanceMonitor.h"
#include "MonotonicClock.h"
// Time reference
extern QElapsedTimer gTimer;
extern ReferenceClock gTimer;
// Executable directory
extern QString gExeDir;
......
......@@ -36,12 +36,8 @@ int main(int argc, char* argv[])
#endif
qInstallMessageHandler(logMessages);
gTimer.start();
logInitBanner();
if (!gTimer.isMonotonic()) // TODO: give the user the option to continue with an unreliable clock
qFatal("Non monotonic reference clock.");
makeFolders();
/*
......
......@@ -3,6 +3,7 @@
#include <memory>
#include <QElapsedTimer>
#include <QFileInfo>
#include <QObject>
#include <QThread>
......
#include "utils.h"
#include <QElapsedTimer>
const char Token::Delimiter = '\t';
const char Token::Newline = '\n';
const char Token::JournalEntry = 'J';
......@@ -52,7 +54,7 @@ void logExitBanner()
logStream.flush();
}
extern QElapsedTimer gTimer;
extern ReferenceClock gTimer;
extern LogWidget* gLogWidget;
void logMessages(QtMsgType type, const QMessageLogContext& context, const QString& msg)
{
......
......@@ -8,7 +8,6 @@
#include <QCameraViewfinderSettings>
#include <QDateTime>
#include <QDebug>
#include <QElapsedTimer>
#include <QFile>
#include <QMutex>
#include <QTextStream>
......@@ -22,7 +21,8 @@ Q_DECLARE_METATYPE(QCameraInfo)
#include "LogWidget.h"
typedef qint64 Timestamp;
#include "MonotonicClock.h"
extern const Timestamp maxTimestamp;
Q_DECLARE_METATYPE(Timestamp)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment