Initial commit

GazeData.h

+#ifndef GAZEDATA_H
+#define GAZEDATA_H
+
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+
+enum Movement {
+    FIXATION = 0,
+    SACCADE = 1,
+    PURSUIT = 2,
+    NOISE = 3,
+    UNDEF = 4
+};
+
+class GazeDataEntry
+{
+public:
+    GazeDataEntry(const double &ts, const double &confidence, const double &x, const double &y) :
+        ts(ts),
+        confidence(confidence),
+        x(x),
+        y(y),
+        v(0),
+        classification(UNDEF)
+    {}
+
+    bool isFixation() { return classification == FIXATION; }
+    bool isSaccade() { return classification == SACCADE; }
+    bool isPursuit() { return classification == PURSUIT; }
+    bool isNoise() { return classification == NOISE; }
+    bool isUndef() { return classification == UNDEF; }
+
+    unsigned int pause();
+
+    double x, y, v;
+    double confidence;
+    double ts;
+    Movement classification;
+};
+
+#endif // GAZEDATA_H

IBDT.cpp

+#include "IBDT.h"
+
+#include <climits>
+#include "opencv2/core.hpp"
+
+using namespace std;
+using namespace cv;
+
+IBDT::IBDT(const double &maxSaccadeDurationMs, const double &minSampleConfidence, const CLASSIFICATION &classification) :
+    maxSaccadeDurationMs(maxSaccadeDurationMs),
+    minSampleConfidence(minSampleConfidence),
+    classification(classification),
+    cur(NULL)
+{
+
+}
+
+double IBDT::estimateVelocity(const GazeDataEntry &cur, const GazeDataEntry &prev)
+{
+    double dist = norm( Point2f(cur.x, cur.y) - Point2f( prev.x, prev.y) );
+    double dt = cur.ts - prev.ts;
+    return dist / dt;
+}
+
+void IBDT::train(std::vector<GazeDataEntry> &gaze)
+{
+    Mat samples;
+
+    // Find first valid sample
+    auto previous = gaze.begin();
+    while (previous != gaze.end() && previous->confidence < minSampleConfidence) {
+        previous->v = std::numeric_limits<double>::quiet_NaN();
+        previous++;
+    }
+
+    // Estimate velocities for remaining training samples
+    for ( auto g = previous+1; g != gaze.end(); g++ ) {
+        if ( g->confidence < minSampleConfidence ){
+            g->v = std::numeric_limits<double>::quiet_NaN();
+            continue;
+        }
+
+        g->v = estimateVelocity( *g, *previous );
+        if (!isnan(g->v))
+            samples.push_back(g->v);
+
+        previous = g;
+    }
+
+    model = ml::EM::create();
+    model->setClustersNumber(2);
+    model->setCovarianceMatrixType(ml::EM::COV_MAT_GENERIC);
+    model->setTermCriteria(TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 15000, 1e-6));
+    model->trainEM(samples);
+
+    fIdx = 0;
+    sIdx = 1;
+    Mat means = model->getMeans();
+    if (means.at<double>(0) > means.at<double>(1)) {
+        fIdx = 1;
+        sIdx = 0;
+    }
+
+    fMean = means.at<double>(fIdx);
+    sMean = means.at<double>(sIdx);
+}
+
+void IBDT::updateFixationAndSaccadeLikelihood()
+{
+    if (cur->v < fMean) {
+        cur->fixation.likelihood = 1;
+        cur->saccade.likelihood = 0;
+        return;
+    }
+
+    if (cur->v > sMean) {
+        cur->fixation.likelihood = 0;
+        cur->saccade.likelihood = 1;
+        return;
+    }
+
+    Mat sample = (Mat_<double>(1,1) << cur->v);
+    Mat likelihoods;
+    model->predict( sample, likelihoods );
+    cur->fixation.likelihood = likelihoods.at<double>(fIdx);
+    cur->saccade.likelihood = likelihoods.at<double>(sIdx);
+}
+
+void IBDT::binaryClassification()
+{
+    if (cur->fixation.likelihood > cur->saccade.likelihood)
+        cur->classification = FIXATION;
+    else
+        cur->classification = SACCADE;
+}
+
+void IBDT::ternaryClassification()
+{
+    // Class that maximizes posterior probability
+    double maxPosterior = cur->fixation.posterior;
+    cur->classification = FIXATION;
+
+    if ( cur->saccade.posterior > maxPosterior ) {
+        cur->classification = SACCADE;
+        maxPosterior = cur->saccade.posterior;
+    }
+
+    if ( cur->pursuit.posterior > maxPosterior )
+        cur->classification = PURSUIT;
+
+    // Catch up saccades as saccades
+    if (cur->v > sMean)
+        cur->classification = SACCADE;
+}
+
+void IBDT::addPoint(GazeDataEntry &entry)
+{
+    entry.classification = UNDEF;
+
+    // Low confidence, ignore it
+    if (entry.confidence < minSampleConfidence)
+        return;
+
+    // Add new point to window and update previous valid point
+    window.push_back( entry );
+    prev = cur;
+    cur = &window.back();
+
+    // Remove old entries from window
+    while (true) {
+        if ( cur->ts - window.front().ts > 2*maxSaccadeDurationMs )
+            window.pop_front();
+        else
+            break;
+    }
+
+    // First point being classified is a special case (since we classify interframe periods)
+    if (!prev) {
+        cur->classification = UNDEF;
+        entry.classification = cur->classification;
+        return;
+    }
+
+    // We have an intersample period, let's classify it
+    cur->v = estimateVelocity(*cur, *prev);
+
+    // Update the priors
+    updatePursuitPrior();
+    cur->saccade.prior = cur->fixation.prior = 1 - cur->pursuit.prior;
+
+    // Update the likelihoods
+    updatePursuitLikelihood();
+    updateFixationAndSaccadeLikelihood();
+
+    // Update the posteriors
+    cur->pursuit.update();
+    cur->fixation.update();
+    cur->saccade.update();
+
+    // Decision
+    switch (classification) {
+        case TERNARY:
+            ternaryClassification();
+            break;
+        case BINARY:
+            binaryClassification();
+            break;
+    }
+
+    entry.classification = cur->classification;
+}
+
+void IBDT::updatePursuitLikelihood()
+{
+    if (window.size() < 2)
+        return;
+
+    double movement = 0;
+    for ( auto d = window.begin()+1; d != window.end() ; d++) {
+        // if (d-v > 0) // original
+        if (d->v > fMean && d->v < sMean) // adaptive: don't activate with too small or too large movements
+            movement++;
+    }
+
+    double n = window.size() - 1 ;
+    double movementRatio = movement / n;
+    cur->pursuit.likelihood = movementRatio;
+}
+
+void IBDT::updatePursuitPrior()
+{
+    vector<double> previousLikelihoods;
+    for ( auto d = window.begin(); d != window.end()-1; d++)
+        previousLikelihoods.push_back(d->pursuit.likelihood);
+    cur->pursuit.prior =
+        accumulate( previousLikelihoods.begin(), previousLikelihoods.end(), 0.0)
+        / previousLikelihoods.size();
+}
+

IBDT.h

+#ifndef IBDT_H
+#define IBDT_H
+
+#include <deque>
+#include <vector>
+#include <algorithm>
+
+#include <GazeData.h>
+
+#include <opencv2/ml.hpp>
+
+class IBDT_Prob {
+public:
+    IBDT_Prob() : prior(0), likelihood(0), posterior(0) {}
+    double prior;
+    double likelihood;
+    double posterior;
+    void update() { posterior = prior*likelihood; }
+};
+
+class IBDT_Data : public GazeDataEntry {
+public:
+    IBDT_Data(GazeDataEntry base) :
+        GazeDataEntry(base),
+        pursuit(),
+        fixation(),
+        saccade() { }
+
+    IBDT_Prob pursuit;
+    IBDT_Prob fixation;
+    IBDT_Prob saccade;
+};
+
+class IBDT
+{
+public:
+    enum CLASSIFICATION {
+        TERNARY = 0,
+        BINARY = 1
+    };
+
+    IBDT(const double &maxSaccadeDurationMs=80, const double &minSampleConfidence=0.5, const enum CLASSIFICATION &classification=TERNARY);
+    void addPoint(GazeDataEntry &entry);
+    void train(std::vector<GazeDataEntry> &gaze);
+    double estimateVelocity(const GazeDataEntry &cur, const GazeDataEntry &prev);
+
+private:
+    double maxSaccadeDurationMs;
+    double minSampleConfidence;
+    CLASSIFICATION classification;
+    std::deque<IBDT_Data> window;
+
+    IBDT_Data *cur;
+    IBDT_Data *prev;
+    bool firstPoint;
+    cv::Ptr<cv::ml::EM> model;
+    unsigned int fIdx, sIdx;
+    double fMean, sMean;
+
+    void updatePursuitPrior();
+    void updatePursuitLikelihood();
+    void updatePursuitLikelihoodNew();
+    void updateFixationAndSaccadeLikelihood();
+
+    void binaryClassification();
+    void ternaryClassification();
+};
+
+#endif // IBDT_H

README

+################################################################################
+# LICENSE
+################################################################################
+
+Copyright (c) 2017, University of Tübingen
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for non-commercial purposes, without fee, and without a written
+agreement is hereby granted, provided that the above copyright notice and this
+paragraph and the following two paragraphs appear in all copies.
+
+IN NO EVENT SHALL THE UNIVERSITY OF TÜBINGEN BE LIABLE TO ANY PARTY FOR DIRECT,
+INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
+ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
+UNIVERSITY OF TÜBINGEN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+UNIVERSITY OF TÜBINGEN SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND
+UNIVERSITY OF TÜBINGEN HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
+UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+
+
+
+################################################################################
+# CITATION
+################################################################################
+
+T. Santini, W. Fuhl, T. Kübler, E. Kasneci
+Bayesian Identification of Fixations, Saccades, and Smooth Pursuits
+ACM Symposium on Eye Tracking Research & Applications, ETRA 2016
+
+
+
+################################################################################
+# USAGE
+################################################################################
+
+This is not a script, but a (quick and dirty) C++ class that implements I-BDT;
+the output is not supposed to match the original Matlab implementation perfectly.
+The class is capable of online classification, and the user can select between
+binary (fixation / saccade) or ternary (fixation / saccade / pursuits)
+classification.
+
+In general:
+
+1) Include the header:
+#include "IBDT.h"
+
+2) Instantiate the class with default or custom parameters:
+
+IBDT* ibdt = new IBDT(); // default parameters (80, 0.5, IBDT::TERNARY)
+IBDT* ibdt = new IBDT(150, 0.9, IBDT::BINARY); // maximum saccade duration = 150ms, minimum sample confidence = 0.9, binary classification
+
+3) Collect samples for parameter estimation on a std::vector<GazeDataEntry> and feed
+it to the ibdt->train function:
+
+std::vector<GazeDataEntry> entries;
+// collect samples for some time; filter if desired
+ibdt->train(entries);
+
+4) After the algorithm is trained, feeding an entry to the algorithm will fill
+the entrie's classification and velocity:
+
+GazeDataEntry entry(timestamp, confidence, x, y);
+ibdt->addPoint( entry );
+cout << entry->v << entry->classification << endl;
+