Commit 5c58bad9 authored by Thiago Santini's avatar Thiago Santini

Finishes first reorganization of the GazeEstimationMethod super class

parent 50e8c687
......@@ -99,7 +99,6 @@ void GazeEstimation::detectOutliers()
t = calibrationTuples[i];
switch (cfg.inputType) {
case GazeEstimationMethod::BINOCULAR:
case GazeEstimationMethod::BINOCULAR_MEAN_POR:
if (!t->lEye.validPupil || !t->rEye.validPupil)
t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
break;
......@@ -129,7 +128,6 @@ void GazeEstimation::detectOutliers()
switch (cfg.inputType) {
case GazeEstimationMethod::BINOCULAR:
case GazeEstimationMethod::BINOCULAR_MEAN_POR:
if (!validLeft || !validRight)
cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
break;
......@@ -175,7 +173,6 @@ void GazeEstimation::detectOutliers()
switch (cfg.inputType) {
case GazeEstimationMethod::BINOCULAR:
case GazeEstimationMethod::BINOCULAR_MEAN_POR:
if (!validLeft || !validRight)
cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
break;
......@@ -274,7 +271,7 @@ void GazeEstimation::calibrate()
return;
}
calibrated = gazeEstimationMethod->calibrate(calibrationInliers, cfg.inputType, error);
calibrated = gazeEstimationMethod->calibrate(calibrationInliers, error);
if (!calibrated) {
qInfo() << error;
......@@ -308,11 +305,10 @@ void GazeEstimation::estimate(DataTuple dataTuple)
if (calibrated) {
// TODO: GazeEstimates in the DataTuple
GazeEstimate left, right, binocular;
gazeEstimationMethod->estimate2d(dataTuple, left, right, binocular);
gazeEstimationMethod->estimate(dataTuple, left, right, binocular);
switch (cfg.inputType) {
case GazeEstimationMethod::BINOCULAR:
case GazeEstimationMethod::BINOCULAR_MEAN_POR:
dataTuple.field.gazeEstimate = to3D(binocular.gp);
dataTuple.field.validGazeEstimate = gazeEstimationMethod->validGazeEstimate(binocular);
break;
......@@ -430,6 +426,19 @@ static void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key
data = QVariant(tokens[idxMap[key]]).value<T>();
}
template <>
static inline void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key, vector<Marker>& markers)
{
if (idxMap.count(key) == 0)
return;
QString markersStr = tokens[idxMap[key]];
for (auto& markerStr : markersStr.split(Token::MarkerEnd)) {
Marker tmp;
if (tmp.parse(markerStr))
markers.push_back(std::move(tmp));
}
}
void GazeEstimation::loadTuplesFromFile(CollectionTuple::TupleType tupleType, QString fileName)
{
QFile inputFile(fileName);
......@@ -449,39 +458,39 @@ void GazeEstimation::loadTuplesFromFile(CollectionTuple::TupleType tupleType, QS
for (int i = 0; i < tokens.size(); i++)
idxMap[tokens[i]] = i;
} else {
addData(idxMap, tokens, "timestamp", tuple.timestamp);
// Field
addData(idxMap, tokens, "field.timestamp", tuple.field.timestamp);
addData(idxMap, tokens, "field.gaze.x", tuple.field.gazeEstimate.x);
addData(idxMap, tokens, "field.gaze.y", tuple.field.gazeEstimate.y);
addData(idxMap, tokens, "field.gaze.z", tuple.field.gazeEstimate.z);
addData(idxMap, tokens, "field.gaze.valid", tuple.field.validGazeEstimate);
addData(idxMap, tokens, "field.collectionMarker.id", tuple.field.collectionMarker.id);
addData(idxMap, tokens, "field.collectionMarker.x", tuple.field.collectionMarker.center.x);
addData(idxMap, tokens, "field.collectionMarker.y", tuple.field.collectionMarker.center.y);
addData(idxMap, tokens, "field.collectionMarker.z", tuple.field.collectionMarker.center.z);
addData(idxMap, tokens, "field.undistorted", tuple.field.undistorted);
addData(idxMap, tokens, "field.width", tuple.field.width);
addData(idxMap, tokens, "field.height", tuple.field.height);
// TODO: read other markers
// Left Eye
addData(idxMap, tokens, "left.timestamp", tuple.lEye.timestamp);
addData(idxMap, tokens, "left.pupil.x", tuple.lEye.pupil.center.x);
addData(idxMap, tokens, "left.pupil.y", tuple.lEye.pupil.center.y);
addData(idxMap, tokens, "left.pupil.width", tuple.lEye.pupil.size.width);
addData(idxMap, tokens, "left.pupil.height", tuple.lEye.pupil.size.height);
addData(idxMap, tokens, "left.pupil.angle", tuple.lEye.pupil.angle);
addData(idxMap, tokens, "left.pupil.valid", tuple.lEye.validPupil);
// Right Eye
addData(idxMap, tokens, "right.timestamp", tuple.rEye.timestamp);
addData(idxMap, tokens, "right.pupil.x", tuple.rEye.pupil.center.x);
addData(idxMap, tokens, "right.pupil.y", tuple.rEye.pupil.center.y);
addData(idxMap, tokens, "right.pupil.width", tuple.rEye.pupil.size.width);
addData(idxMap, tokens, "right.pupil.height", tuple.rEye.pupil.size.height);
addData(idxMap, tokens, "right.pupil.angle", tuple.rEye.pupil.angle);
addData(idxMap, tokens, "right.pupil.valid", tuple.rEye.validPupil);
addData(idxMap, tokens, "sync.timestamp", tuple.timestamp);
auto addFieldData = [&](const QString& prefix, FieldData& field) {
addData(idxMap, tokens, prefix + "timestamp", field.timestamp);
addData(idxMap, tokens, prefix + "gaze.x", field.gazeEstimate.x);
addData(idxMap, tokens, prefix + "gaze.y", field.gazeEstimate.y);
addData(idxMap, tokens, prefix + "gaze.z", field.gazeEstimate.z);
addData(idxMap, tokens, prefix + "gaze.valid", field.validGazeEstimate);
addData(idxMap, tokens, prefix + "collectionMarker.id", field.collectionMarker.id);
addData(idxMap, tokens, prefix + "collectionMarker.x", field.collectionMarker.center.x);
addData(idxMap, tokens, prefix + "collectionMarker.y", field.collectionMarker.center.y);
addData(idxMap, tokens, prefix + "collectionMarker.z", field.collectionMarker.center.z);
addData(idxMap, tokens, prefix + "undistorted", field.undistorted);
addData(idxMap, tokens, prefix + "width", field.width);
addData(idxMap, tokens, prefix + "height", field.height);
addData(idxMap, tokens, prefix + "markers", field.markers);
addData(idxMap, tokens, prefix + "processingTime", field.processingTimestamp);
};
addFieldData("field.", tuple.field);
auto addEyeData = [&](const QString& prefix, EyeData& eye) {
addData(idxMap, tokens, prefix + "timestamp", eye.timestamp);
addData(idxMap, tokens, prefix + "pupil.x", eye.pupil.center.x);
addData(idxMap, tokens, prefix + "pupil.y", eye.pupil.center.y);
addData(idxMap, tokens, prefix + "pupil.width", eye.pupil.size.width);
addData(idxMap, tokens, prefix + "pupil.height", eye.pupil.size.height);
addData(idxMap, tokens, prefix + "pupil.angle", eye.pupil.angle);
addData(idxMap, tokens, prefix + "pupil.confidence", eye.pupil.confidence);
addData(idxMap, tokens, prefix + "pupil.valid", eye.validPupil);
addData(idxMap, tokens, prefix + "pupil.processingTimestamp", eye.validPupil);
};
addEyeData("left.", tuple.lEye);
addEyeData("right.", tuple.rEye);
// Add
tuple.tupleType = tupleType;
......
......@@ -85,7 +85,7 @@ public:
, pupilPositionOutliers(true)
, pupilOutlineOutliers(true)
, pupilDiameterOutliers(true)
, inputType(GazeEstimationMethod::BINOCULAR_MEAN_POR)
, inputType(GazeEstimationMethod::BINOCULAR)
, gazeEstimationMethod("POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY")
, visualize(true)
, visualizationTimeS(5)
......
......@@ -60,8 +60,7 @@ GazeEstimationWidget::GazeEstimationWidget(QString id, QWidget* parent)
ui->collectionTypeComboBox->addItem("Evaluation", CollectionTuple::EVALUATION);
ui->inputTypeBox->blockSignals(true);
ui->inputTypeBox->addItem("Binocular (Mean POR)", GazeEstimationMethod::BINOCULAR_MEAN_POR);
ui->inputTypeBox->addItem("Binocular (Mean Pupil)", GazeEstimationMethod::BINOCULAR);
ui->inputTypeBox->addItem("Binocular", GazeEstimationMethod::BINOCULAR);
ui->inputTypeBox->addItem("Mono Left", GazeEstimationMethod::MONO_LEFT);
ui->inputTypeBox->addItem("Mono Right", GazeEstimationMethod::MONO_RIGHT);
for (int i = 0; i < ui->inputTypeBox->count(); i++)
......
......@@ -38,6 +38,7 @@ public:
% prefix % journalField(QStringLiteral("processingTime"));
}
private:
void toQStringImpl(QString& str) const override
{
str.append(journalField(timestamp));
......
......@@ -52,6 +52,7 @@ public:
% prefix % journalField(QStringLiteral("processingTime"));
}
private:
void toQStringImpl(QString& str) const override
{
str.append(journalField(timestamp));
......
#ifndef MARKER_H
#define MARKER_H
#include <QRegularExpression>
#include <QString>
#include <opencv2/opencv.hpp>
......@@ -35,6 +36,20 @@ public:
cachedStrSize = std::max(cachedStrSize, tmp.size());
return tmp;
}
bool parse(const QString& s)
{
if (!s.isEmpty()) {
QRegularExpressionMatch match = re.match(s);
if (match.hasMatch()) {
id = match.captured(1).toInt();
center.x = match.captured(2).toFloat();
center.y = match.captured(3).toFloat();
center.z = match.captured(4).toFloat();
return true;
}
}
return false;
}
std::vector<cv::Point2f> corners;
cv::Point3f center;
......@@ -49,6 +64,7 @@ private:
QString strWithPrecision(const double v) const { return QString::number(v, 'g', Token::Precision); }
QString strWithPrecision(const float v) const { return strWithPrecision(static_cast<double>(v)); }
QString valueStr() const { return QString("%2%1%3%1%4").arg(Token::MarkerValueDelimiter).arg(strWithPrecision(center.x)).arg(strWithPrecision(center.y)).arg(strWithPrecision(center.z)); }
static const QRegularExpression re;
};
#endif // MARKER_H
......@@ -17,3 +17,4 @@ static const int gCollectionTupleTypeId = qRegisterMetaType<CollectionTuple::Tup
static const int gDataTupleId = qRegisterMetaType<DataTuple>("DataTuple");
int Marker::cachedStrSize = 0;
const QRegularExpression Marker::re("(\\d+):(\\d+.?\\d+)x(\\d+.?\\d+)x(\\d+.?\\d+)");
......@@ -8,16 +8,14 @@ void GazeEstimationMethod::estimateBinocular2d(GazeEstimate& left, GazeEstimate&
if (leftValid && rightValid) {
binocular.gp = 0.5 * (left.gp + right.gp);
binocular.valid = true;
binocular.pupilConfidence = left.pupilConfidence + right.pupilConfidence;
binocular.pupilConfidence = 0.5 * (left.pupilConfidence + right.pupilConfidence);
} else {
// TODO: right now we just use the one that is valid.
// maybe use historical data to improve it in the future?
if (leftValid) {
binocular.gp = left.gp;
}
if (rightValid) {
binocular.gp = right.gp;
}
if (leftValid)
binocular = left;
if (rightValid)
binocular = right;
}
}
......@@ -35,6 +33,9 @@ cv::Point2f GazeEstimationMethod::estimate2d(const DataTuple& tuple, GazeEstimat
void GazeEstimationMethod::estimateBinocular3d(GazeEstimate& left, GazeEstimate& right, GazeEstimate& binocular)
{
// TODO
(void)left;
(void)right;
(void)binocular;
}
cv::Point3f GazeEstimationMethod::estimate3d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right, GazeEstimate& binocular)
{
......
......@@ -36,24 +36,15 @@ public:
virtual ~GazeEstimationMethod() {}
enum InputType {
BINOCULAR = 0,
BINOCULAR_MEAN_POR = 1,
BINOCULAR = 1,
MONO_LEFT = 2,
MONO_RIGHT = 3,
UNKNOWN = 4
};
// TODO: this really needs to be reorganized. Main points:
// 1) Get rid of inputType and let the methods figure out what they CAN do
// 2) Make the interface aware of left and right gaze estimations instead of a single one
// 3) Make a virtual method to estimate the binocular gaze estimate -- e.g., by taking the mean point or estimating the depth + parallax
virtual bool calibrate(std::vector<CollectionTuple>& calibrationTuples, InputType inputType, QString& error) = 0;
virtual cv::Point3f estimateGaze(const DataTuple& tuple, const InputType inputType) = 0;
virtual std::string description() = 0;
cv::Point2f estimate2d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right, GazeEstimate& binocular);
virtual bool calibrate(std::vector<CollectionTuple>& calibrationTuples, QString& error) = 0;
void estimate(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right, GazeEstimate& binocular);
cv::Point2f estimate2d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right, GazeEstimate& binocular);
float minPupilConfidence;
......
......@@ -4,104 +4,26 @@ using namespace std;
using namespace cv;
Homography::Homography()
: binocularCalibrated(false)
, monoLeftCalibrated(false)
, monoRightCalibrated(false)
: leftCalibrated(false)
, rightCalibrated(false)
{
}
bool Homography::calibrate(vector<CollectionTuple>& calibrationTuples, InputType inputType, QString& errorMsg)
bool Homography::calibrate(vector<CollectionTuple>& calibrationTuples, QString& errorMsg)
{
vector<Point2f> leftPupil;
vector<Point2f> rightPupil;
vector<Point2f> meanPupil;
vector<Point2f> gaze;
for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
leftPupil.push_back(calibrationTuples[i].lEye.pupil.center);
rightPupil.push_back(calibrationTuples[i].rEye.pupil.center);
meanPupil.push_back((calibrationTuples[i].lEye.pupil.center + calibrationTuples[i].rEye.pupil.center) / 2.0);
gaze.push_back(to2D(calibrationTuples[i].field.collectionMarker.center));
}
// Estimate parameters for all, independently of input type; this allows us
// to use a single eye later even if mean pupil was selected for calibration
leftCalibrated = calibrate(leftPupil, gaze, lH, errorMsg);
rightCalibrated = calibrate(rightPupil, gaze, rH, errorMsg);
binocularCalibrated = calibrate(meanPupil, gaze, mH, errorMsg);
monoLeftCalibrated = calibrate(leftPupil, gaze, lH, errorMsg);
monoRightCalibrated = calibrate(rightPupil, gaze, rH, errorMsg);
switch (inputType) {
case BINOCULAR:
return binocularCalibrated;
case BINOCULAR_MEAN_POR:
return monoLeftCalibrated && monoRightCalibrated;
case MONO_LEFT:
return monoLeftCalibrated;
case MONO_RIGHT:
return monoRightCalibrated;
}
calibrationInputType = inputType;
return true;
}
Point3f Homography::estimateGaze(const DataTuple& tuple, const InputType inputType)
{
Point3f estimate(-1, -1, -1);
vector<Point2d> ev;
vector<Point2d> pupil;
switch (inputType) {
case BINOCULAR:
if (!binocularCalibrated)
return estimate;
pupil.push_back((tuple.lEye.pupil.center + tuple.rEye.pupil.center) / 2);
perspectiveTransform(pupil, ev, mH);
estimate = to3D(ev[0]);
break;
case BINOCULAR_MEAN_POR:
if (!monoLeftCalibrated || !monoRightCalibrated)
return estimate;
pupil.push_back(tuple.lEye.pupil.center);
perspectiveTransform(pupil, ev, lH);
estimate = to3D(ev[0]);
pupil.clear();
pupil.push_back(tuple.rEye.pupil.center);
perspectiveTransform(pupil, ev, rH);
estimate = (estimate + to3D(ev[0])) / 2.0;
break;
case MONO_LEFT:
if (!monoLeftCalibrated)
return estimate;
pupil.push_back(tuple.lEye.pupil.center);
perspectiveTransform(pupil, ev, lH);
estimate = to3D(ev[0]);
break;
case MONO_RIGHT:
if (!monoRightCalibrated)
return estimate;
pupil.push_back(tuple.rEye.pupil.center);
perspectiveTransform(pupil, ev, rH);
estimate = to3D(ev[0]);
break;
default:
break;
}
return estimate;
}
Point2f Homography::evaluate(Point2f leftPupil, Point2f rightPupil, InputType inputType)
{
(void)leftPupil;
(void)rightPupil;
(void)inputType;
return Point2f();
return leftCalibrated || rightCalibrated;
}
bool Homography::calibrate(vector<Point2f> pupil, vector<Point2f> gaze, Mat& H, QString& errorMsg)
......@@ -128,7 +50,6 @@ bool Homography::calibrate(vector<Point2f> pupil, vector<Point2f> gaze, Mat& H,
void Homography::implEstimate2d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right)
{
auto estimate = [&](const bool& calibrated, const Pupil& pupil, const Mat& H, GazeEstimate& ge) {
if (calibrated) {
vector<Point2d> pv;
......@@ -140,6 +61,6 @@ void Homography::implEstimate2d(const DataTuple& tuple, GazeEstimate& left, Gaze
}
};
estimate(monoLeftCalibrated, tuple.lEye.pupil, lH, left);
estimate(monoRightCalibrated, tuple.rEye.pupil, rH, right);
estimate(leftCalibrated, tuple.lEye.pupil, lH, left);
estimate(rightCalibrated, tuple.rEye.pupil, rH, right);
}
......@@ -13,23 +13,10 @@
class Homography : public GazeEstimationMethod {
public:
Homography();
bool calibrate(std::vector<CollectionTuple>& calibrationTuples, InputType inputType, QString& errorMsg) override;
cv::Point3f estimateGaze(const DataTuple& tuple, const InputType inputType) override;
bool allowCenterAndScale() { return true; }
bool calibrate(std::vector<CollectionTuple>& calibrationTuples, QString& errorMsg) override;
std::string description() override { return "Homography"; }
private:
InputType calibrationInputType;
cv::Mat mH;
cv::Mat lH;
cv::Mat rH;
bool binocularCalibrated;
bool monoLeftCalibrated;
bool monoRightCalibrated;
bool calibrate(std::vector<cv::Point2f> pupil, std::vector<cv::Point2f> gaze, cv::Mat& H, QString& errorMsg);
cv::Point2f evaluate(cv::Point2f leftPupil, cv::Point2f rightPupil, InputType inputType);
bool has3d() override { return false; }
void implEstimate2d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right) override;
void implEstimate3d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right) override
......@@ -38,6 +25,16 @@ private:
(void)left;
(void)right;
}
// Implementation related
bool leftCalibrated;
bool rightCalibrated;
cv::Mat mH;
cv::Mat lH;
cv::Mat rH;
bool calibrate(std::vector<cv::Point2f> pupil, std::vector<cv::Point2f> gaze, cv::Mat& H, QString& errorMsg);
};
#endif // HOMOGRAPHY_H
......@@ -5,9 +5,8 @@ using namespace std;
using namespace cv;
PolyFit::PolyFit(PlType plType)
: binocularCalibrated(false)
, monoLeftCalibrated(false)
, monoRightCalibrated(false)
: leftCalibrated(false)
, rightCalibrated(false)
{
this->plType = plType;
......@@ -54,21 +53,15 @@ string PolyFit::description()
}
}
PolyFit::~PolyFit()
{
}
bool PolyFit::calibrate(vector<CollectionTuple>& calibrationTuples, InputType inputType, QString& errorMsg)
bool PolyFit::calibrate(vector<CollectionTuple>& calibrationTuples, QString& errorMsg)
{
PointVector leftPupil;
PointVector rightPupil;
PointVector meanPupil;
PointVector gaze;
for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
leftPupil.insert(calibrationTuples[i].lEye.pupil.center);
rightPupil.insert(calibrationTuples[i].rEye.pupil.center);
meanPupil.insert((calibrationTuples[i].lEye.pupil.center + calibrationTuples[i].rEye.pupil.center) / 2.0);
gaze.insert(calibrationTuples[i].field.collectionMarker.center);
}
......@@ -77,120 +70,27 @@ bool PolyFit::calibrate(vector<CollectionTuple>& calibrationTuples, InputType in
return false;
}
// Estimate parameters for all, independently of input type; this allows us
// to use a single eye later even if mean pupil was selected for calibration
binocularCalibrated = calibrate(meanPupil.x, meanPupil.y, gaze.x, mcx, errorMsg)
&& calibrate(meanPupil.x, meanPupil.y, gaze.y, mcy, errorMsg);
monoLeftCalibrated = calibrate(leftPupil.x, leftPupil.y, gaze.x, lcx, errorMsg)
leftCalibrated = calibrate(leftPupil.x, leftPupil.y, gaze.x, lcx, errorMsg)
&& calibrate(leftPupil.x, leftPupil.y, gaze.y, lcy, errorMsg);
monoRightCalibrated = calibrate(rightPupil.x, rightPupil.y, gaze.x, rcx, errorMsg)
rightCalibrated = calibrate(rightPupil.x, rightPupil.y, gaze.x, rcx, errorMsg)
&& calibrate(rightPupil.x, rightPupil.y, gaze.y, rcy, errorMsg);
switch (inputType) {
case BINOCULAR:
return binocularCalibrated;
case BINOCULAR_MEAN_POR:
return monoLeftCalibrated && monoRightCalibrated;
case MONO_LEFT:
return monoLeftCalibrated;
case MONO_RIGHT:
return monoRightCalibrated;
}
calibrationInputType = inputType;
buildDepthMap(calibrationTuples);
return true;
return leftCalibrated || rightCalibrated;
//buildDepthMap(calibrationTuples);
}
Point3f PolyFit::estimateGaze(const DataTuple& tuple, const InputType inputType)
{
Point2f estimate = evaluate(tuple.lEye.pupil.center, tuple.rEye.pupil.center, inputType);
Point3f estimate3D;
estimate3D.x = estimate.x;
estimate3D.y = estimate.y;
if (estimate.x < 0 || estimate.x >= depthMap.cols || estimate.y < 0 || estimate.y >= depthMap.rows)
estimate3D.z = -1;
else
estimate3D.z = depthMap.at<float>(estimate.y, estimate.x);
return estimate3D;
}
Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType inputType)
void PolyFit::implEstimate2d(const DataTuple& tuple, GazeEstimate& left, GazeEstimate& right)
{
Point2f estimate(-1, -1);
switch (inputType) {
case BINOCULAR:
if (!binocularCalibrated)
return estimate;
estimate.x = evaluate(
(leftPupil.x + rightPupil.x) / 2.0,
(leftPupil.y + rightPupil.y) / 2.0,
mcx);
estimate.y = evaluate(
(leftPupil.x + rightPupil.x) / 2.0,
(leftPupil.y + rightPupil.y) / 2.0,
mcy);
break;
case BINOCULAR_MEAN_POR:
if (!monoLeftCalibrated || !monoRightCalibrated)
return estimate;
estimate.x = evaluate(
leftPupil.x,
leftPupil.y,
lcx)
+ evaluate(
rightPupil.x,
rightPupil.y,
rcx);
estimate.x /= 2.0;
estimate.y = evaluate(
rightPupil.x,
rightPupil.y,
rcy)
+ evaluate(
leftPupil.x,
leftPupil.y,
lcy);
estimate.y /= 2.0;
break;