Commit fc141ade authored by Thiago Santini's avatar Thiago Santini

Initial commit

parents
#ifndef GAZEDATA_H
#define GAZEDATA_H
#include <vector>
#include "opencv2/opencv.hpp"
enum Movement {
FIXATION = 0,
SACCADE = 1,
PURSUIT = 2,
NOISE = 3,
UNDEF = 4
};
class GazeDataEntry
{
public:
GazeDataEntry(const double &ts, const double &confidence, const double &x, const double &y) :
ts(ts),
confidence(confidence),
x(x),
y(y),
v(0),
classification(UNDEF)
{}
bool isFixation() { return classification == FIXATION; }
bool isSaccade() { return classification == SACCADE; }
bool isPursuit() { return classification == PURSUIT; }
bool isNoise() { return classification == NOISE; }
bool isUndef() { return classification == UNDEF; }
unsigned int pause();
double x, y, v;
double confidence;
double ts;
Movement classification;
};
#endif // GAZEDATA_H
#include "IBDT.h"
#include <climits>
#include "opencv2/core.hpp"
using namespace std;
using namespace cv;
IBDT::IBDT(const double &maxSaccadeDurationMs, const double &minSampleConfidence, const CLASSIFICATION &classification) :
maxSaccadeDurationMs(maxSaccadeDurationMs),
minSampleConfidence(minSampleConfidence),
classification(classification),
cur(NULL)
{
}
double IBDT::estimateVelocity(const GazeDataEntry &cur, const GazeDataEntry &prev)
{
double dist = norm( Point2f(cur.x, cur.y) - Point2f( prev.x, prev.y) );
double dt = cur.ts - prev.ts;
return dist / dt;
}
void IBDT::train(std::vector<GazeDataEntry> &gaze)
{
Mat samples;
// Find first valid sample
auto previous = gaze.begin();
while (previous != gaze.end() && previous->confidence < minSampleConfidence) {
previous->v = std::numeric_limits<double>::quiet_NaN();
previous++;
}
// Estimate velocities for remaining training samples
for ( auto g = previous+1; g != gaze.end(); g++ ) {
if ( g->confidence < minSampleConfidence ){
g->v = std::numeric_limits<double>::quiet_NaN();
continue;
}
g->v = estimateVelocity( *g, *previous );
if (!isnan(g->v))
samples.push_back(g->v);
previous = g;
}
model = ml::EM::create();
model->setClustersNumber(2);
model->setCovarianceMatrixType(ml::EM::COV_MAT_GENERIC);
model->setTermCriteria(TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 15000, 1e-6));
model->trainEM(samples);
fIdx = 0;
sIdx = 1;
Mat means = model->getMeans();
if (means.at<double>(0) > means.at<double>(1)) {
fIdx = 1;
sIdx = 0;
}
fMean = means.at<double>(fIdx);
sMean = means.at<double>(sIdx);
}
void IBDT::updateFixationAndSaccadeLikelihood()
{
if (cur->v < fMean) {
cur->fixation.likelihood = 1;
cur->saccade.likelihood = 0;
return;
}
if (cur->v > sMean) {
cur->fixation.likelihood = 0;
cur->saccade.likelihood = 1;
return;
}
Mat sample = (Mat_<double>(1,1) << cur->v);
Mat likelihoods;
model->predict( sample, likelihoods );
cur->fixation.likelihood = likelihoods.at<double>(fIdx);
cur->saccade.likelihood = likelihoods.at<double>(sIdx);
}
void IBDT::binaryClassification()
{
if (cur->fixation.likelihood > cur->saccade.likelihood)
cur->classification = FIXATION;
else
cur->classification = SACCADE;
}
void IBDT::ternaryClassification()
{
// Class that maximizes posterior probability
double maxPosterior = cur->fixation.posterior;
cur->classification = FIXATION;
if ( cur->saccade.posterior > maxPosterior ) {
cur->classification = SACCADE;
maxPosterior = cur->saccade.posterior;
}
if ( cur->pursuit.posterior > maxPosterior )
cur->classification = PURSUIT;
// Catch up saccades as saccades
if (cur->v > sMean)
cur->classification = SACCADE;
}
void IBDT::addPoint(GazeDataEntry &entry)
{
entry.classification = UNDEF;
// Low confidence, ignore it
if (entry.confidence < minSampleConfidence)
return;
// Add new point to window and update previous valid point
window.push_back( entry );
prev = cur;
cur = &window.back();
// Remove old entries from window
while (true) {
if ( cur->ts - window.front().ts > 2*maxSaccadeDurationMs )
window.pop_front();
else
break;
}
// First point being classified is a special case (since we classify interframe periods)
if (!prev) {
cur->classification = UNDEF;
entry.classification = cur->classification;
return;
}
// We have an intersample period, let's classify it
cur->v = estimateVelocity(*cur, *prev);
// Update the priors
updatePursuitPrior();
cur->saccade.prior = cur->fixation.prior = 1 - cur->pursuit.prior;
// Update the likelihoods
updatePursuitLikelihood();
updateFixationAndSaccadeLikelihood();
// Update the posteriors
cur->pursuit.update();
cur->fixation.update();
cur->saccade.update();
// Decision
switch (classification) {
case TERNARY:
ternaryClassification();
break;
case BINARY:
binaryClassification();
break;
}
entry.classification = cur->classification;
}
void IBDT::updatePursuitLikelihood()
{
if (window.size() < 2)
return;
double movement = 0;
for ( auto d = window.begin()+1; d != window.end() ; d++) {
// if (d-v > 0) // original
if (d->v > fMean && d->v < sMean) // adaptive: don't activate with too small or too large movements
movement++;
}
double n = window.size() - 1 ;
double movementRatio = movement / n;
cur->pursuit.likelihood = movementRatio;
}
void IBDT::updatePursuitPrior()
{
vector<double> previousLikelihoods;
for ( auto d = window.begin(); d != window.end()-1; d++)
previousLikelihoods.push_back(d->pursuit.likelihood);
cur->pursuit.prior =
accumulate( previousLikelihoods.begin(), previousLikelihoods.end(), 0.0)
/ previousLikelihoods.size();
}
#ifndef IBDT_H
#define IBDT_H
#include <deque>
#include <vector>
#include <algorithm>
#include <GazeData.h>
#include <opencv2/ml.hpp>
class IBDT_Prob {
public:
IBDT_Prob() : prior(0), likelihood(0), posterior(0) {}
double prior;
double likelihood;
double posterior;
void update() { posterior = prior*likelihood; }
};
class IBDT_Data : public GazeDataEntry {
public:
IBDT_Data(GazeDataEntry base) :
GazeDataEntry(base),
pursuit(),
fixation(),
saccade() { }
IBDT_Prob pursuit;
IBDT_Prob fixation;
IBDT_Prob saccade;
};
class IBDT
{
public:
enum CLASSIFICATION {
TERNARY = 0,
BINARY = 1
};
IBDT(const double &maxSaccadeDurationMs=80, const double &minSampleConfidence=0.5, const enum CLASSIFICATION &classification=TERNARY);
void addPoint(GazeDataEntry &entry);
void train(std::vector<GazeDataEntry> &gaze);
double estimateVelocity(const GazeDataEntry &cur, const GazeDataEntry &prev);
private:
double maxSaccadeDurationMs;
double minSampleConfidence;
CLASSIFICATION classification;
std::deque<IBDT_Data> window;
IBDT_Data *cur;
IBDT_Data *prev;
bool firstPoint;
cv::Ptr<cv::ml::EM> model;
unsigned int fIdx, sIdx;
double fMean, sMean;
void updatePursuitPrior();
void updatePursuitLikelihood();
void updatePursuitLikelihoodNew();
void updateFixationAndSaccadeLikelihood();
void binaryClassification();
void ternaryClassification();
};
#endif // IBDT_H
################################################################################
# LICENSE
################################################################################
Copyright (c) 2017, University of Tübingen
Permission to use, copy, modify, and distribute this software and its
documentation for non-commercial purposes, without fee, and without a written
agreement is hereby granted, provided that the above copyright notice and this
paragraph and the following two paragraphs appear in all copies.
IN NO EVENT SHALL THE UNIVERSITY OF TÜBINGEN BE LIABLE TO ANY PARTY FOR DIRECT,
INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
UNIVERSITY OF TÜBINGEN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
UNIVERSITY OF TÜBINGEN SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND
UNIVERSITY OF TÜBINGEN HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
################################################################################
# CITATION
################################################################################
T. Santini, W. Fuhl, T. Kübler, E. Kasneci
Bayesian Identification of Fixations, Saccades, and Smooth Pursuits
ACM Symposium on Eye Tracking Research & Applications, ETRA 2016
################################################################################
# USAGE
################################################################################
This is not a script, but a (quick and dirty) C++ class that implements I-BDT;
the output is not supposed to match the original Matlab implementation perfectly.
The class is capable of online classification, and the user can select between
binary (fixation / saccade) or ternary (fixation / saccade / pursuits)
classification.
In general:
1) Include the header:
#include "IBDT.h"
2) Instantiate the class with default or custom parameters:
IBDT* ibdt = new IBDT(); // default parameters (80, 0.5, IBDT::TERNARY)
IBDT* ibdt = new IBDT(150, 0.9, IBDT::BINARY); // maximum saccade duration = 150ms, minimum sample confidence = 0.9, binary classification
3) Collect samples for parameter estimation on a std::vector<GazeDataEntry> and feed
it to the ibdt->train function:
std::vector<GazeDataEntry> entries;
// collect samples for some time; filter if desired
ibdt->train(entries);
4) After the algorithm is trained, feeding an entry to the algorithm will fill
the entrie's classification and velocity:
GazeDataEntry entry(timestamp, confidence, x, y);
ibdt->addPoint( entry );
cout << entry->v << entry->classification << endl;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment