Commit ab379fda authored by Thiago Santini's avatar Thiago Santini

Starts merging from the PuRe and tracking repositories

Adds generic confidence measure
Adds generic coarse pupil location estimation
Adapts pupil detector interfaces to work with ROIs
parent f4b4ff45
......@@ -40,7 +40,8 @@ SOURCES +=\
$${TOP}/src/LogWidget.cpp \
$${TOP}/src/PerformanceMonitor.cpp \
$${TOP}/src/PerformanceMonitorWidget.cpp \
src/CameraCalibration.cpp
$${TOP}/src/CameraCalibration.cpp \
$${TOP}/src/pupil-detection/PupilDetectionMethod.cpp
HEADERS += \
$${TOP}/src/MainWindow.h\
......@@ -67,7 +68,7 @@ HEADERS += \
$${TOP}/src/LogWidget.h \
$${TOP}/src/PerformanceMonitor.h \
$${TOP}/src/PerformanceMonitorWidget.h \
src/CameraCalibration.h
$${TOP}/src/CameraCalibration.h
FORMS += \
$${TOP}/src/MainWindow.ui \
......
......@@ -66,48 +66,61 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
data.input = frame;
}
if (cfg.flip != CV_FLIP_NONE)
flip(data.input, data.input, cfg.flip);
if (data.input.channels() > 1) // TODO: make it algorithm dependent
cvtColor(data.input, data.input, CV_BGR2GRAY);
data.pupil = RotatedRect();
data.validPupil = false;
if (pupilDetectionMethod != NULL) {
Rect tmp = Rect(
Point(sROI.x() * data.input.cols, sROI.y() * data.input.rows),
Point( eROI.x() * data.input.cols, eROI.y() * data.input.rows)
);
Point start = tmp.tl();
Point end = tmp.br();
Mat roi = data.input(Rect(start, end));
if (cfg.processingDownscalingFactor > 1) {
resize(roi, roi, Size(),
1/cfg.processingDownscalingFactor,
1/cfg.processingDownscalingFactor,
INTER_AREA);
if (cfg.flip != CV_FLIP_NONE)
flip(data.input, data.input, cfg.flip);
if (data.input.channels() > 1) // TODO: make it algorithm dependent
cvtColor(data.input, data.input, CV_BGR2GRAY);
data.pupil = Pupil();
data.validPupil = false;
if (pupilDetectionMethod != NULL) {
Rect userROI = Rect(
Point(sROI.x() * data.input.cols, sROI.y() * data.input.rows),
Point( eROI.x() * data.input.cols, eROI.y() * data.input.rows)
);
float scalingFactor = 1;
if (cfg.processingDownscalingFactor > 1)
scalingFactor = 1.0 / cfg.processingDownscalingFactor;
/*
* From here on, our reference frame is the scaled user ROI
*/
Mat downscaled;
resize(data.input(userROI), downscaled, Size(),
scalingFactor, scalingFactor,
INTER_AREA);
Rect coarseROI = {0, 0, downscaled.cols, downscaled.rows };
// If the user wants a coarse location and the method has none embedded,
// we further constrain the search using the generic one
if (!pupilDetectionMethod->hasCoarseLocation() && cfg.coarseDetection) {
coarseROI = PupilDetectionMethod::coarsePupilDetection( downscaled );
data.coarseROI = Rect(
userROI.tl() + coarseROI.tl() / scalingFactor,
userROI.tl() + coarseROI.br() / scalingFactor
);
} else
data.coarseROI = Rect();
// Actual detection
if (coarseROI.width > 10 && coarseROI.height > 10) { // minimum size otherwise some algorithms might crash
pupilDetectionMethod->run( downscaled, coarseROI, data.pupil );
if ( ! pupilDetectionMethod->hasConfidence() )
data.pupil.confidence = PupilDetectionMethod::outlineContrastConfidence(downscaled, data.pupil);
}
if (roi.rows > 10 && roi.cols > 10) { // minimum size otherwise some algorithms might crash
if (roi.rows <= 640 && roi.cols <= 640) // TODO: fix ExCuSe and ElSe size limit
data.pupil = pupilDetectionMethod->run(roi);
}
if (data.pupil.center.x > 0 && data.pupil.center.y > 0) {
// Upscale
data.pupil.resize( 1.0 / scalingFactor );
// User region shift
data.pupil.shift( userROI.tl() );
data.validPupil = true;
}
if (data.pupil.center.x > 0 && data.pupil.center.y > 0) {
if (cfg.processingDownscalingFactor > 1) {
data.pupil.center.x *= cfg.processingDownscalingFactor;
data.pupil.center.y *= cfg.processingDownscalingFactor;
data.pupil.size.width *= cfg.processingDownscalingFactor;
data.pupil.size.height *= cfg.processingDownscalingFactor;
}
data.pupil.center.x += start.x;
data.pupil.center.y += start.y;
data.validPupil = true;
}
}
data.processingTimestamp = gTimer.elapsed() - data.timestamp;
......
......@@ -35,14 +35,15 @@ public:
explicit EyeData(){
timestamp = 0;
input = cv::Mat();
pupil = cv::RotatedRect(cv::Point2f(0,0), cv::Size2f(0,0), 0);
pupil = Pupil();
validPupil = false;
processingTimestamp = 0;
}
cv::Mat input;
cv::RotatedRect pupil;
bool validPupil;
Pupil pupil;
bool validPupil;
cv::Rect coarseROI;
// TODO: header, toQString, and the reading from file (see the Calibration class) should be unified
// to avoid placing things in the wrong order / with the wrong string
......@@ -60,9 +61,11 @@ public:
tmp.append(gDataSeparator);
tmp.append(prefix + "pupil.angle");
tmp.append(gDataSeparator);
tmp.append(prefix + "pupil.valid");
tmp.append(prefix + "pupil.confidence");
tmp.append(gDataSeparator);
tmp.append(prefix + "pupil.valid");
tmp.append(gDataSeparator);
tmp.append(prefix + "processingTimestamp");
tmp.append(prefix + "processingTime");
tmp.append(gDataSeparator);
return tmp;
}
......@@ -79,7 +82,9 @@ public:
tmp.append(gDataSeparator);
tmp.append(QString::number(pupil.size.height));
tmp.append(gDataSeparator);
tmp.append(QString::number(pupil.angle));
tmp.append(QString::number(pupil.angle));
tmp.append(gDataSeparator);
tmp.append(QString::number(pupil.confidence));
tmp.append(gDataSeparator);
tmp.append(QString::number(validPupil));
tmp.append(gDataSeparator);
......@@ -99,14 +104,16 @@ public:
:
inputSize(cv::Size(0, 0)),
flip(CV_FLIP_NONE),
undistort(false),
undistort(false),
coarseDetection(true),
processingDownscalingFactor(2),
pupilDetectionMethod(ElSe::desc.c_str())
{}
cv::Size inputSize;
CVFlip flip;
bool undistort;
bool undistort;
bool coarseDetection;
double processingDownscalingFactor;
QString pupilDetectionMethod;
......@@ -117,7 +124,8 @@ public:
settings->setValue("height", inputSize.height);
settings->setValue("flip", flip);
settings->setValue("undistort", undistort);
settings->setValue("processingDownscalingFactor", processingDownscalingFactor);
settings->setValue("coarseDetection", coarseDetection);
settings->setValue("processingDownscalingFactor", processingDownscalingFactor);
settings->setValue("pupilDetectionMethod", pupilDetectionMethod);
}
......@@ -128,7 +136,8 @@ public:
set(settings, "height", inputSize.height);
set(settings, "flip", flip);
set(settings, "undistort", undistort);
set(settings, "processingDownscalingFactor", processingDownscalingFactor);
set(settings, "coarseDetection", coarseDetection);
set(settings, "processingDownscalingFactor", processingDownscalingFactor);
set(settings, "pupilDetectionMethod", pupilDetectionMethod);
}
};
......@@ -192,13 +201,17 @@ public:
layout->addWidget(box);
hBoxLayout = new QHBoxLayout();
box = new QGroupBox("Pupil Detection");
box->setWhatsThis("Selects pupil detection method.");
formLayout = new QFormLayout();
box = new QGroupBox("Pupil Detection");
box->setWhatsThis("Selects pupil detection method.");
box->setToolTip(box->whatsThis());
box->setLayout(hBoxLayout);
pupilDetectionComboBox = new QComboBox();
hBoxLayout->addWidget(pupilDetectionComboBox);
box->setLayout(formLayout);
coarseDetectionBox = new QCheckBox();
coarseDetectionBox->setWhatsThis("Estimate a coarse location for the pupil location prior to detection.");
coarseDetectionBox->setToolTip(box->whatsThis());
formLayout->addRow( new QLabel("Coarse Detection:"), coarseDetectionBox );
pupilDetectionComboBox = new QComboBox();
formLayout->addRow(pupilDetectionComboBox);
layout->addWidget(box);
applyButton = new QPushButton("Apply");
......@@ -223,7 +236,8 @@ public slots:
widthSB->setValue(cfg.inputSize.width);
heightSB->setValue(cfg.inputSize.height);
downscalingSB->setValue(cfg.processingDownscalingFactor);
downscalingSB->setValue(cfg.processingDownscalingFactor);
coarseDetectionBox->setChecked(cfg.coarseDetection);
for (int i=0; i<flipComboBox->count(); i++)
if (flipComboBox->itemData(i).toInt() == cfg.flip)
flipComboBox->setCurrentIndex(i);
......@@ -238,8 +252,9 @@ public slots:
EyeImageProcessorConfig cfg;
cfg.inputSize.width = widthSB->value();
cfg.inputSize.height = heightSB->value();
cfg.processingDownscalingFactor = downscalingSB->value();
cfg.flip = (CVFlip) flipComboBox->currentData().toInt();
cfg.processingDownscalingFactor = downscalingSB->value();
cfg.flip = (CVFlip) flipComboBox->currentData().toInt();
cfg.coarseDetection = coarseDetectionBox->isChecked();
cfg.pupilDetectionMethod = pupilDetectionComboBox->currentData().toString();
cfg.save(settings);
emit updateConfig();
......@@ -248,9 +263,10 @@ public slots:
private:
QPushButton *applyButton;
QSpinBox *widthSB, *heightSB;
QCheckBox *undistortBox;
QComboBox *flipComboBox;
QDoubleSpinBox *downscalingSB;
QCheckBox *undistortBox;
QCheckBox *coarseDetectionBox;
QComboBox *flipComboBox;
QDoubleSpinBox *downscalingSB;
};
class EyeImageProcessor : public QObject
......
......@@ -102,7 +102,7 @@ public:
tmp.append(gDataSeparator);
tmp.append(prefix + "markers");
tmp.append(gDataSeparator);
tmp.append(prefix + "processingTimestamp");
tmp.append(prefix + "processingTime");
tmp.append(gDataSeparator);
return tmp;
}
......
......@@ -7,6 +7,8 @@
using namespace cv;
std::string ElSe::desc = "ElSe (Fuhl et al. 2016)";
float ElSe::minArea = 0;
float ElSe::maxArea = 0;
#define IMG_SIZE 680 //400
......@@ -283,8 +285,8 @@ static std::vector<std::vector<Point>> get_curves(Mat *pic, Mat *edge, Mat *magn
if(add_curve){ // pupil area
if(ellipse.size.width*ellipse.size.height < pic->cols*pic->rows*0.005 ||
ellipse.size.width*ellipse.size.height > pic->cols*pic->rows*0.2)//0.1
if(ellipse.size.width*ellipse.size.height < ElSe::minArea ||
ellipse.size.width*ellipse.size.height > ElSe::maxArea)
add_curve=false;
}
......@@ -1328,6 +1330,12 @@ if(pos.y>0 && pos.y<pic->rows && pos.x>0 && pos.x<pic->cols){
RotatedRect ElSe::run(const Mat &frame)
{
RotatedRect ellipse;
Point pos(0,0);
if (frame.rows > IMG_SIZE || frame.cols > IMG_SIZE)
return ellipse;
Mat pic;
normalize(frame, pic, 0, 255, NORM_MINMAX, CV_8U);
......@@ -1335,8 +1343,6 @@ RotatedRect ElSe::run(const Mat &frame)
double mean_dist=3;
int inner_color_range=0;
RotatedRect ellipse;
Point pos(0,0);
int start_x=(int)floor(double(pic.cols)*border);
int start_y=(int)floor(double(pic.rows)*border);
......@@ -1374,3 +1380,23 @@ RotatedRect ElSe::run(const Mat &frame)
return ellipse;
}
void ElSe::run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
{
if (roi.area() < 10) {
qWarning() << "Bad ROI: falling back to regular detection.";
PupilDetectionMethod::run(frame, pupil);
return;
}
if (minPupilDiameterPx > 0 && maxPupilDiameterPx > 0 ) {
minArea = pow(minPupilDiameterPx,2);
maxArea = pow(maxPupilDiameterPx,2);
} else {
minArea = frame.cols * frame.rows * 0.005;
maxArea = frame.cols * frame.rows * 0.2;
}
pupil = run( frame(roi) );
if (pupil.center.x > 0 && pupil.center.y > 0)
pupil.shift( roi.tl() );
}
......@@ -20,8 +20,13 @@ class ElSe : public PupilDetectionMethod
public:
ElSe() { mDesc = desc; }
cv::RotatedRect run(const cv::Mat &frame);
bool hasPupilOutline() { return true; }
static std::string desc;
void run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
bool hasConfidence() { return false; }
bool hasCoarseLocation() { return false; }
static std::string desc;
static float minArea;
static float maxArea;
};
#endif // ELSE_H
......@@ -1720,19 +1720,29 @@ else{
}
RotatedRect ExCuSe::run(const Mat &frame)
{
if (frame.rows > IMG_SIZE || frame.cols > IMG_SIZE)
return RotatedRect();
Mat target;
normalize(frame, target, 0, 255, NORM_MINMAX, CV_8U);
Mat pic_th = Mat::zeros(target.rows, target.cols, CV_8U);
Mat th_edges = Mat::zeros(target.rows, target.cols, CV_8U);
return runexcuse(&target, &pic_th, &th_edges, 15);
}
void ExCuSe::run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
{
if (roi.area() < 10) {
qWarning() << "Bad ROI: falling back to regular detection.";
PupilDetectionMethod::run(frame, pupil);
return;
}
(void) minPupilDiameterPx;
(void) maxPupilDiameterPx;
pupil = run( frame(roi) );
if (pupil.center.x > 0 && pupil.center.y > 0)
pupil.shift( roi.tl() );
}
......@@ -20,8 +20,10 @@ class ExCuSe : public PupilDetectionMethod
public:
ExCuSe() { mDesc = desc;}
cv::RotatedRect run(const cv::Mat &frame);
bool hasPupilOutline() { return true; }
static std::string desc;
void run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
bool hasConfidence() { return false; }
bool hasCoarseLocation() { return false; }
static std::string desc;
};
#endif // EXCUSE_H
#include "PupilDetectionMethod.h"
#include <QDebug>
using namespace std;
using namespace cv;
//#define DBG_COARSE_PUPIL_DETECTION
//#define DBG_OUTLINE_CONTRAST
Rect PupilDetectionMethod::coarsePupilDetection(const Mat &frame, const float &minCoverage, const int &workingWidth, const int &workingHeight)
{
// We can afford to work on a very small input for haar features, but retain the aspect ratio
float xr = frame.cols / (float) workingWidth;
float yr = frame.rows / (float) workingHeight;
float r = min( xr, yr );
Mat downscaled;
resize(frame, downscaled, Size(), 1/r, 1/r, CV_INTER_LINEAR);
int ystep = (int) max<float>( 0.01f*downscaled.rows, 1.0f);
int xstep = (int) max<float>( 0.01f*downscaled.cols, 1.0f);
float d = (float) sqrt( pow(downscaled.rows, 2) + pow(downscaled.cols, 2) );
// Pupil radii is based on PuRe assumptions
int min_r = (int) (0.5 * 0.07 * d);
int max_r = (int) (0.5 * 0.29 * d);
int r_step = (int) max<float>( 0.1f*(max_r + min_r), 1.0f);
// TODO: padding so we consider the borders as well!
/* Haar-like feature suggested by Swirski. For details, see
* Świrski, Lech, Andreas Bulling, and Neil Dodgson.
* "Robust real-time pupil tracking in highly off-axis images."
* Proceedings of the Symposium on Eye Tracking Research and Applications. ACM, 2012.
*
* However, we collect a per-pixel maxima instead of the global one
*/
Mat itg;
integral(downscaled, itg, CV_32S);
Mat res = Mat::zeros( downscaled.rows, downscaled.cols, CV_32F);
float best_response = std::numeric_limits<float>::min();
deque< pair<Rect, float> > candidates;
for (int r = min_r; r<=max_r; r+=r_step) {
int step = 3*r;
Point ia, ib, ic, id;
Point oa, ob, oc, od;
int inner_count = (2*r) * (2*r);
int outer_count = (2*step)*(2*step) - inner_count;
float inner_norm = 1.0f / (255*inner_count);
float outer_norm = 1.0f / (255*outer_count);
for (int y = step; y<downscaled.rows-step; y+=ystep) {
oa.y = y - step;
ob.y = y - step;
oc.y = y + step;
od.y = y + step;
ia.y = y - r;
ib.y = y - r;
ic.y = y + r;
id.y = y + r;
for (int x = step; x<downscaled.cols-step; x+=xstep) {
oa.x = x - step;
ob.x = x + step;
oc.x = x + step;
od.x = x - step;
ia.x = x - r;
ib.x = x + r;
ic.x = x + r;
id.x = x - r;
int inner = itg.ptr<int>(ic.y)[ic.x] + itg.ptr<int>(ia.y)[ia.x] -
itg.ptr<int>(ib.y)[ib.x] - itg.ptr<int>(id.y)[id.x];
int outer = itg.ptr<int>(oc.y)[oc.x] + itg.ptr<int>(oa.y)[oa.x] -
itg.ptr<int>(ob.y)[ob.x] - itg.ptr<int>(od.y)[od.x] - inner;
float inner_mean = inner_norm*inner;
float outer_mean = outer_norm*outer;
float response = (outer_mean - inner_mean);
if (response > best_response)
best_response = response;
if ( response > res.ptr<float>(y)[x] ) {
res.ptr<float>(y)[x] = response;
// The pupil is too small, the padding too large; we combine them.
candidates.push_back( make_pair(Rect( 0.5*(ia+oa), 0.5*(ic+oc) ), response) );
}
}
}
}
// Eliminate low response candidates then sort and combine
for ( auto c = candidates.begin(); c != candidates.end();) {
if ( c->second < 0.5 * best_response)
c = candidates.erase(c);
else
c++;
}
auto compare = [] (const pair<Rect, float> &a, const pair<Rect,float> &b) {
return (a.second > b.second);
};
sort( candidates.begin(), candidates.end(), compare);
#ifdef DBG_COARSE_PUPIL_DETECTION
Mat dbg;
cvtColor(downscaled, dbg, CV_GRAY2BGR);
#endif
// Now add until we reach the minimum coverage or run out of candidates
Rect coarse;
int minWidth = minCoverage * downscaled.cols;
int minHeight = minCoverage * downscaled.rows;
for ( int i=0; i<candidates.size(); i++ ) {
auto &c = candidates[i];
if (coarse.area() == 0)
coarse = c.first;
else
coarse |= c.first;
#ifdef DBG_COARSE_PUPIL_DETECTION
rectangle(dbg, candidates[i].first, Scalar(0,255,255));
#endif
if (coarse.width > minWidth && coarse.height > minHeight)
break;
}
#ifdef DBG_COARSE_PUPIL_DETECTION
rectangle(dbg, coarse, Scalar(0,255,0));
resize(dbg, dbg, Size(), r, r);
imshow("Coarse Detection Debug", dbg);
#endif
// Upscale result
coarse.x *= r;
coarse.y *= r;
coarse.width *= r;
coarse.height *= r;
// Sanity test
Rect imRoi = Rect(0, 0, frame.cols, frame.rows);
coarse &= imRoi;
if (coarse.area() == 0)
return imRoi;
return coarse;
}
static const float sinTable[] = {
0.0000000f , 0.0174524f , 0.0348995f , 0.0523360f , 0.0697565f , 0.0871557f ,
0.1045285f , 0.1218693f , 0.1391731f , 0.1564345f , 0.1736482f , 0.1908090f ,
0.2079117f , 0.2249511f , 0.2419219f , 0.2588190f , 0.2756374f , 0.2923717f ,
0.3090170f , 0.3255682f , 0.3420201f , 0.3583679f , 0.3746066f , 0.3907311f ,
0.4067366f , 0.4226183f , 0.4383711f , 0.4539905f , 0.4694716f , 0.4848096f ,
0.5000000f , 0.5150381f , 0.5299193f , 0.5446390f , 0.5591929f , 0.5735764f ,
0.5877853f , 0.6018150f , 0.6156615f , 0.6293204f , 0.6427876f , 0.6560590f ,
0.6691306f , 0.6819984f , 0.6946584f , 0.7071068f , 0.7193398f , 0.7313537f ,
0.7431448f , 0.7547096f , 0.7660444f , 0.7771460f , 0.7880108f , 0.7986355f ,
0.8090170f , 0.8191520f , 0.8290376f , 0.8386706f , 0.8480481f , 0.8571673f ,
0.8660254f , 0.8746197f , 0.8829476f , 0.8910065f , 0.8987940f , 0.9063078f ,
0.9135455f , 0.9205049f , 0.9271839f , 0.9335804f , 0.9396926f , 0.9455186f ,
0.9510565f , 0.9563048f , 0.9612617f , 0.9659258f , 0.9702957f , 0.9743701f ,
0.9781476f , 0.9816272f , 0.9848078f , 0.9876883f , 0.9902681f , 0.9925462f ,
0.9945219f , 0.9961947f , 0.9975641f , 0.9986295f , 0.9993908f , 0.9998477f ,
1.0000000f , 0.9998477f , 0.9993908f , 0.9986295f , 0.9975641f , 0.9961947f ,
0.9945219f , 0.9925462f , 0.9902681f , 0.9876883f , 0.9848078f , 0.9816272f ,
0.9781476f , 0.9743701f , 0.9702957f , 0.9659258f , 0.9612617f , 0.9563048f ,
0.9510565f , 0.9455186f , 0.9396926f , 0.9335804f , 0.9271839f , 0.9205049f ,
0.9135455f , 0.9063078f , 0.8987940f , 0.8910065f , 0.8829476f , 0.8746197f ,
0.8660254f , 0.8571673f , 0.8480481f , 0.8386706f , 0.8290376f , 0.8191520f ,
0.8090170f , 0.7986355f , 0.7880108f , 0.7771460f , 0.7660444f , 0.7547096f ,
0.7431448f , 0.7313537f , 0.7193398f , 0.7071068f , 0.6946584f , 0.6819984f ,
0.6691306f , 0.6560590f , 0.6427876f , 0.6293204f , 0.6156615f , 0.6018150f ,
0.5877853f , 0.5735764f , 0.5591929f , 0.5446390f , 0.5299193f , 0.5150381f ,
0.5000000f , 0.4848096f , 0.4694716f , 0.4539905f , 0.4383711f , 0.4226183f ,
0.4067366f , 0.3907311f , 0.3746066f , 0.3583679f , 0.3420201f , 0.3255682f ,</