Commit 3ae7a8e0 authored by Thiago Santini's avatar Thiago Santini

Starts pupil tracking interface

parent 39810f78
......@@ -44,7 +44,10 @@ SOURCES +=\
$${TOP}/src/pupil-detection/PupilDetectionMethod.cpp \
$${TOP}/src/Overlay.cpp \
$${TOP}/src/CommandManager.cpp \
$${TOP}/src/ERWidget.cpp
$${TOP}/src/ERWidget.cpp \
src/pupil-tracking/PupiTtrackingMethod.cpp \
src/pupil-tracking/PuReTy.cpp \
src/pupil-detection/PuRe.cpp
HEADERS += \
$${TOP}/src/MainWindow.h\
......@@ -74,7 +77,10 @@ HEADERS += \
$${TOP}/src/CameraCalibration.h \
$${TOP}/src/Overlay.h \
$${TOP}/src/CommandManager.h \
$${TOP}/src/ERWidget.h
$${TOP}/src/ERWidget.h \
$${TOP}/src/pupil-tracking/PupilTrackingMethod.h \
src/pupil-tracking/PuReTy.h \
src/pupil-detection/PuRe.h
FORMS += \
$${TOP}/src/MainWindow.ui \
......
......@@ -176,8 +176,4 @@ private slots:
void reset();
};
Q_DECLARE_METATYPE(cv::Mat)
Q_DECLARE_METATYPE(QCameraInfo)
Q_DECLARE_METATYPE(Timestamp)
#endif // CAMERA_H
......@@ -8,12 +8,14 @@ static int gEyeDataId = qRegisterMetaType<EyeData>("EyeData");
EyeImageProcessor::EyeImageProcessor(QString id, QObject *parent)
: id(id),
pupilDetectionMethod(NULL),
sROI(QPointF(0,0)),
pupilTrackingMethod(NULL),
sROI(QPointF(0,0)),
eROI(QPointF(1,1)),
QObject(parent)
{
availablePupilDetectionMethods.push_back(new ElSe());
availablePupilDetectionMethods.push_back(new ExCuSe());
availablePupilDetectionMethods.push_back(new PuRe());
availablePupilDetectionMethods.push_back(new ElSe());
availablePupilDetectionMethods.push_back(new ExCuSe());
#ifdef STARBURST
availablePupilDetectionMethods.push_back(new Starburst());
#endif
......@@ -23,7 +25,9 @@ EyeImageProcessor::EyeImageProcessor(QString id, QObject *parent)
settings = new QSettings(gCfgDir + "/" + id + " ImageProcessor", QSettings::IniFormat);
updateConfig();
pmIdx = gPerformanceMonitor.enrol(id, "Image Processor");
pmIdx = gPerformanceMonitor.enrol(id, "Image Processor");
pupilTrackingMethod = new PuReTy();
}
void EyeImageProcessor::updateConfig()
......@@ -97,7 +101,7 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
// If the user wants a coarse location and the method has none embedded,
// we further constrain the search using the generic one
if (!pupilDetectionMethod->hasCoarseLocation() && cfg.coarseDetection) {
coarseROI = PupilDetectionMethod::coarsePupilDetection( downscaled );
coarseROI = PupilDetectionMethod::coarsePupilDetection( downscaled, 0.5f );
data.coarseROI = Rect(
userROI.tl() + coarseROI.tl() / scalingFactor,
userROI.tl() + coarseROI.br() / scalingFactor
......@@ -105,12 +109,15 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
} else
data.coarseROI = Rect();
// Actual detection
if (coarseROI.width > 10 && coarseROI.height > 10) { // minimum size otherwise some algorithms might crash
bool tracking = true;
if (tracking && pupilTrackingMethod) {
pupilTrackingMethod->run(timestamp, downscaled, coarseROI, data.pupil, *pupilDetectionMethod);
} else {
pupilDetectionMethod->run( downscaled, coarseROI, data.pupil );
// TODO: expose this to the user
if ( ! pupilDetectionMethod->hasConfidence() )
data.pupil.confidence = PupilDetectionMethod::outlineContrastConfidence(downscaled, data.pupil);
}
}
if (data.pupil.center.x > 0 && data.pupil.center.y > 0) {
// Upscale
......
......@@ -18,6 +18,7 @@
#include "InputWidget.h"
#include "pupil-detection/PuRe.h"
#include "pupil-detection/ElSe.h"
#include "pupil-detection/ExCuSe.h"
#ifdef STARBURST
......@@ -28,6 +29,9 @@
#endif
#include "pupil-detection/PupilDetectionMethod.h"
#include "pupil-tracking/PuReTy.h"
#include "pupil-tracking/PupilTrackingMethod.h"
#include "utils.h"
class EyeData : public InputData {
......@@ -94,7 +98,7 @@ public:
}
};
Q_DECLARE_METATYPE(EyeData)
Q_DECLARE_METATYPE(EyeData);
class EyeImageProcessorConfig
......@@ -107,7 +111,7 @@ public:
undistort(false),
coarseDetection(true),
processingDownscalingFactor(2),
pupilDetectionMethod(ElSe::desc.c_str())
pupilDetectionMethod(PuRe::desc.c_str())
{}
cv::Size inputSize;
......@@ -282,7 +286,7 @@ signals:
void newData(EyeData data);
public slots:
void process(Timestamp t, const cv::Mat &frame);
void process(Timestamp t, const cv::Mat &frame);
void updateConfig();
void newROI(QPointF sROI, QPointF eROI);
......@@ -294,6 +298,7 @@ private:
QPointF sROI, eROI;
PupilDetectionMethod *pupilDetectionMethod;
PupilTrackingMethod *pupilTrackingMethod;
unsigned int pmIdx;
};
......
......@@ -150,7 +150,7 @@ public:
}
};
Q_DECLARE_METATYPE(FieldData)
Q_DECLARE_METATYPE(FieldData);
class FieldImageProcessorConfig
{
......
......@@ -1381,7 +1381,7 @@ RotatedRect ElSe::run(const Mat &frame)
return ellipse;
}
void ElSe::run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
void ElSe::run(const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
{
if (roi.area() < 10) {
qWarning() << "Bad ROI: falling back to regular detection.";
......
......@@ -20,7 +20,7 @@ class ElSe : public PupilDetectionMethod
public:
ElSe() { mDesc = desc; }
cv::RotatedRect run(const cv::Mat &frame);
void run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
void run(const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
bool hasConfidence() { return false; }
bool hasCoarseLocation() { return false; }
static std::string desc;
......
......@@ -1731,7 +1731,7 @@ RotatedRect ExCuSe::run(const Mat &frame)
return runexcuse(&target, &pic_th, &th_edges, 15);
}
void ExCuSe::run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
void ExCuSe::run(const cv::Mat &frame, const Rect &roi, Pupil &pupil, const float &minPupilDiameterPx, const float &maxPupilDiameterPx)
{
if (roi.area() < 10) {
qWarning() << "Bad ROI: falling back to regular detection.";
......
......@@ -20,7 +20,7 @@ class ExCuSe : public PupilDetectionMethod
public:
ExCuSe() { mDesc = desc;}
cv::RotatedRect run(const cv::Mat &frame);
void run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
void run(const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1);
bool hasConfidence() { return false; }
bool hasCoarseLocation() { return false; }
static std::string desc;
......
#ifndef PUPILDETECTIONMETHOD_H
#define PUPILDETECTIONMETHOD_H
#include <QMetaType>
#include <QDebug>
#include <string>
#include <deque>
......@@ -52,11 +55,14 @@ public:
confidence > confidenceThreshold;
}
bool hasOutline() const {
return size.width > 0 && size.height > 0;
}
bool hasOutline() const { return size.width > 0 && size.height > 0; }
int majorAxis() const { return std::max<int>(size.width, size.height); }
int minorAxis() const { return std::min<int>(size.width, size.height); }
int diameter() const { return majorAxis(); }
};
Q_DECLARE_METATYPE(Pupil);
class PupilDetectionMethod
{
public:
......@@ -73,21 +79,31 @@ public:
pupil = run(frame);
pupil.confidence = 1;
}
virtual void run(const cv::Mat &frame, const cv::Rect roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1) {
virtual void run(const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1) {
(void) roi;
(void) minPupilDiameterPx;
(void) maxPupilDiameterPx;
run(frame, pupil);
}
// Pupil detection interface used in the tracking; uses an homogeneous confidence measure
Pupil runWithConfidence(const cv::Mat &frame, const cv::Rect &roi, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1) {
Pupil pupil;
run(frame, roi, pupil, minPupilDiameterPx, maxPupilDiameterPx);
pupil.confidence = outlineContrastConfidence(frame, pupil);
return pupil;
}
// Generic coarse pupil detection
static cv::Rect coarsePupilDetection(const cv::Mat &frame, const float &minCoverage=0.5f, const int &workingWidth=80, const int &workingHeight=60);
// Generic confidence metrics
static float outlineContrastConfidence(const cv::Mat &frame, const Pupil &pupil, const int &bias=5);
//Pupil test(const cv::Mat &frame, const cv::Rect &roi, Pupil pupil) { return pupil; }
protected:
std::string mDesc;
};
#endif // PUPILDETECTIONMETHOD_H
#include "PupilTrackingMethod.h"
#include <QtConcurrent/QtConcurrent>
#include <QFuture>
using namespace std;
using namespace cv;
void PupilTrackingMethod::reset()
{
previousPupils.clear();
previousPupil = TrackedPupil();
}
void PupilTrackingMethod::run(const Timestamp &ts, const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, PupilDetectionMethod &pupilDetectionMethod)
{
cv::Size frameSize = { frame.cols, frame.rows };
if (expectedFrameSize != frameSize ) {
// Reference frame changed. Let's start over!
expectedFrameSize = frameSize;
reset();
}
// Remove old samples
while (!previousPupils.empty()) {
if (ts - previousPupils.front().ts > maxAge)
previousPupils.pop_front();
else
break;
}
int minPupilDiameterPx = -1;
int maxPupilDiameterPx = -1;
if ( ! previousPupils.empty() ) {
float meanDiameterPx = std::accumulate( previousPupils.begin(), previousPupils.end(), 0,
[](int sum, const TrackedPupil &p) { return sum + max<int>( p.size.width, p.size.height ); } ) / (float) previousPupils.size();
minPupilDiameterPx = 0.75 * meanDiameterPx;
maxPupilDiameterPx = 1.25 * meanDiameterPx;
}
if ( previousPupil.confidence == NO_CONFIDENCE ) {
pupil = pupilDetectionMethod.runWithConfidence(frame, roi, minPupilDiameterPx, maxPupilDiameterPx);
if (previousPupils.size() > 15) {
if (pupil.diameter() <= maxPupilDiameterPx)
updatePreviousPupil(ts, pupil);
else
pupil.clear();
} else
updatePreviousPupil(ts, pupil);
return;
}
QFuture<Pupil> future;
if (ts - lastDetection > maxTrackingWithoutDetectionTime) {
float previousDiameter = max<float>( previousPupil.size.width, previousPupil.size.height );
future = QtConcurrent::run(&pupilDetectionMethod, &PupilDetectionMethod::runWithConfidence, frame, roi, 0.9*previousDiameter, 1.1*previousDiameter);
parallelDetection = true;
}
run(frame, roi, previousPupil, pupil);
if (parallelDetection) {
parallelDetection = false;
lastDetection = ts;
Pupil detectedPupil = future.result();
if (detectedPupil.confidence > pupil.confidence)
pupil = detectedPupil;
} else {
if ( pupil.confidence < minDetectionConfidence)
pupil = pupilDetectionMethod.runWithConfidence(frame, roi);
}
if ( pupil.confidence > minDetectionConfidence) {
if (previousPupils.size() > 15) {
if (pupil.diameter() <= maxPupilDiameterPx)
updatePreviousPupil(ts, pupil);
else
pupil.clear();
} else
updatePreviousPupil(ts, pupil);
}
/*
// Track
Pupil previousPupil = previousPupils.back();
run(frame, previousPupil, pupil);
if ( ! shouldTrack(pupil, minTrackConfidence) ) {
//qDebug() << pupil.confidence << pupil.valid();
// Tracking failed, run detection
//qDebug() << "tracking failed, detect";
detectAndTrack(ts, frame, pupil, pupilDetectionMethod);
return;
}
redetect(ts, frame, pupil, pupilDetectionMethod);
//qDebug() << (redetect(ts, frame, pupil, pupilDetectionMethod) ? "redetected" : "tracked");
track(ts, pupil);
*/
}
#ifndef PUPILTRACKINGMETHOD_H
#define PUPILTRACKINGMETHOD_H
#include <string>
#include <deque>
#include <QFuture>
#include <opencv2/core.hpp>
#include "pupil-detection/PupilDetectionMethod.h"
#include "utils.h"
class TrackedPupil : public Pupil
{
public:
TrackedPupil(const Timestamp &ts, const Pupil &pupil) :
Pupil(pupil),
ts(ts)
{}
TrackedPupil() :
Pupil(),
ts(0)
{}
Timestamp ts;
};
class PupilTrackingMethod
{
public:
PupilTrackingMethod() {}
~PupilTrackingMethod() {}
// Tracking and detection logic
void run(const Timestamp &ts, const cv::Mat &frame, const cv::Rect &roi, Pupil &pupil, PupilDetectionMethod &pupilDetectionMethod);
// Tracking implementation
virtual void run(const cv::Mat &frame, const cv::Rect &roi, const Pupil &previousPupil, Pupil &pupil, const float &minPupilDiameterPx=-1, const float &maxPupilDiameterPx=-1) = 0;
std::string description() { return mDesc; }
private:
protected:
std::string mDesc;
cv::Size expectedFrameSize = {0, 0};
TrackedPupil previousPupil;
std::deque<TrackedPupil> previousPupils;
Timestamp maxAge = 1500;
Timestamp maxTrackingWithoutDetectionTime = 5000;
Timestamp lastDetection;
bool parallelDetection = false;
float minDetectionConfidence = 0.7f;
float minTrackConfidence = 0.9f;
void updatePreviousPupil( const Timestamp &ts, const Pupil &pupil ) {
if (pupil.confidence > minDetectionConfidence) {
previousPupil = TrackedPupil(ts, pupil);
previousPupils.push_back( previousPupil );
} else
previousPupil = TrackedPupil();
}
void reset();
};
#endif // PUPILTRACKINGMETHOD_H
......@@ -15,12 +15,18 @@
#include <QTextStream>
#include <QMutex>
#include <QCameraViewfinderSettings>
#include <QCameraInfo>
Q_DECLARE_METATYPE(QCameraInfo)
#include <QWidget>
#include <QSettings>
#include <QSoundEffect>
#include <opencv/cv.h>
Q_DECLARE_METATYPE(cv::Rect);
Q_DECLARE_METATYPE(cv::Mat);
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
......@@ -35,6 +41,7 @@ extern QElapsedTimer gTimer;
typedef qint64 Timestamp;
extern Timestamp maxTimestamp;
Q_DECLARE_METATYPE(Timestamp)
extern QString gExeDir;
extern QString gCfgDir;
......@@ -103,4 +110,5 @@ extern std::vector<QString> gLogBuffer;
extern PerformanceMonitor gPerformanceMonitor;
#endif // UTILS_H
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment