Commit f3c1e19e authored by Thiago Santini's avatar Thiago Santini

Estimate gaze when calibrated with binocular but only one pupil is valid

parent 4c2e9478
......@@ -259,10 +259,37 @@ void GazeEstimation::estimate(DataTuple dataTuple)
if (!gazeEstimationMethod)
return;
// TODO: check pupil validity, use single pupil if one of the two is invalid
if (calibrated) {
dataTuple.field.gazeEstimate = gazeEstimationMethod->estimateGaze(dataTuple, cfg.inputType);
dataTuple.field.validGazeEstimate = true;
GazeEstimationMethod::InputType inputType = cfg.inputType;
bool lValid = dataTuple.lEye.pupil.center.x > 0 && dataTuple.lEye.pupil.center.y > 0;
bool rValid = dataTuple.rEye.pupil.center.x > 0 && dataTuple.rEye.pupil.center.y > 0;
switch(inputType) {
case GazeEstimationMethod::BINOCULAR:
case GazeEstimationMethod::BINOCULAR_MEAN_POR:
if (!lValid && !rValid) // None valid
break;
if (lValid && !rValid) // Use only left
inputType = GazeEstimationMethod::MONO_LEFT;
if (rValid && !lValid) // Use only right
inputType = GazeEstimationMethod::MONO_RIGHT;
dataTuple.field.gazeEstimate = gazeEstimationMethod->estimateGaze(dataTuple, inputType);
dataTuple.field.validGazeEstimate = true;
break;
case GazeEstimationMethod::MONO_LEFT:
if (!lValid)
break;
dataTuple.field.gazeEstimate = gazeEstimationMethod->estimateGaze(dataTuple, inputType);
dataTuple.field.validGazeEstimate = true;
break;
case GazeEstimationMethod::MONO_RIGHT:
if (!rValid)
break;
dataTuple.field.gazeEstimate = gazeEstimationMethod->estimateGaze(dataTuple, inputType);
dataTuple.field.validGazeEstimate = true;
break;
}
}
drawGazeEstimationInfo(dataTuple);
......
......@@ -3,7 +3,10 @@
using namespace std;
using namespace cv;
Homography::Homography()
Homography::Homography() :
binocularCalibrated(false),
monoLeftCalibrated(false),
monoRightCalibrated(false)
{
}
......@@ -21,21 +24,22 @@ bool Homography::calibrate(vector<CollectionTuple> &calibrationTuples, InputType
gaze.push_back( to2D(calibrationTuples[i].field.collectionMarker.center) );
}
switch (inputType) {
case BINOCULAR:
if (!calibrate(meanPupil, gaze, mH, errorMsg))
return false;
break;
// Estimate parameters for all, independently of input type; this allows us
// to use a single eye later even if mean pupil was selected for calibration
case UNKNOWN:
return false;
binocularCalibrated = calibrate(meanPupil, gaze, mH, errorMsg);
monoLeftCalibrated = calibrate(leftPupil, gaze, lH, errorMsg);
monoRightCalibrated = calibrate(rightPupil, gaze, rH, errorMsg);
default:
if (!calibrate(leftPupil, gaze, lH, errorMsg))
return false;
if (!calibrate(rightPupil, gaze, rH, errorMsg))
return false;
break;
switch(inputType) {
case BINOCULAR:
return binocularCalibrated;
case BINOCULAR_MEAN_POR:
return monoLeftCalibrated && monoRightCalibrated;
case MONO_LEFT:
return monoLeftCalibrated;
case MONO_RIGHT:
return monoRightCalibrated;
}
calibrationInputType = inputType;
......@@ -46,20 +50,21 @@ Point3f Homography::estimateGaze(const CollectionTuple &tuple, const InputType i
{
Point3f estimate(-1,-1,-1);
if (inputType != calibrationInputType)
return estimate;
vector<Point2d> ev;
vector<Point2d> pupil;
switch (calibrationInputType) {
switch (inputType) {
case BINOCULAR:
if (!binocularCalibrated)
return estimate;
pupil.push_back( (tuple.lEye.pupil.center + tuple.rEye.pupil.center)/2 );
perspectiveTransform(pupil, ev, mH);
estimate = to3D(ev[0]);
break;
case BINOCULAR_MEAN_POR:
if (!monoLeftCalibrated || !monoRightCalibrated)
return estimate;
pupil.push_back( tuple.lEye.pupil.center );
perspectiveTransform(pupil, ev, lH);
estimate = to3D(ev[0]);
......@@ -70,12 +75,16 @@ Point3f Homography::estimateGaze(const CollectionTuple &tuple, const InputType i
break;
case MONO_LEFT:
if (!monoLeftCalibrated)
return estimate;
pupil.push_back( tuple.lEye.pupil.center );
perspectiveTransform(pupil, ev, lH);
estimate = to3D(ev[0]);
break;
case MONO_RIGHT:
if (!monoRightCalibrated)
return estimate;
pupil.push_back( tuple.rEye.pupil.center );
perspectiveTransform(pupil, ev, rH);
estimate = to3D(ev[0]);
......
......@@ -25,6 +25,9 @@ private:
cv::Mat mH;
cv::Mat lH;
cv::Mat rH;
bool binocularCalibrated;
bool monoLeftCalibrated;
bool monoRightCalibrated;
bool calibrate(std::vector<cv::Point2f> pupil, std::vector<cv::Point2f> gaze, cv::Mat &H, QString &errorMsg);
};
......
......@@ -4,7 +4,10 @@
using namespace std;
using namespace cv;
PolyFit::PolyFit(PlType plType)
PolyFit::PolyFit(PlType plType) :
binocularCalibrated(false),
monoLeftCalibrated(false),
monoRightCalibrated(false)
{
this->plType = plType;
......@@ -58,8 +61,6 @@ PolyFit::~PolyFit()
bool PolyFit::calibrate(vector<CollectionTuple> &calibrationTuples, InputType inputType, QString &errorMsg)
{
calibrationInputType = UNKNOWN;
PointVector leftPupil;
PointVector rightPupil;
PointVector meanPupil;
......@@ -77,27 +78,24 @@ bool PolyFit::calibrate(vector<CollectionTuple> &calibrationTuples, InputType in
return false;
}
switch (inputType) {
case BINOCULAR:
if (!calibrate(meanPupil.x, meanPupil.y, gaze.x, mcx, errorMsg))
return false;
if (!calibrate(meanPupil.x, meanPupil.y, gaze.y, mcy, errorMsg))
return false;
break;
// Estimate parameters for all, independently of input type; this allows us
// to use a single eye later even if mean pupil was selected for calibration
binocularCalibrated = calibrate(meanPupil.x, meanPupil.y, gaze.x, mcx, errorMsg)
&& calibrate(meanPupil.x, meanPupil.y, gaze.y, mcy, errorMsg);
monoLeftCalibrated = calibrate(leftPupil.x, leftPupil.y, gaze.x, lcx, errorMsg)
&& calibrate(leftPupil.x, leftPupil.y, gaze.y, lcy, errorMsg);
monoRightCalibrated = calibrate(rightPupil.x, rightPupil.y, gaze.x, rcx, errorMsg)
&& calibrate(rightPupil.x, rightPupil.y, gaze.y, rcy, errorMsg);
case UNKNOWN:
return false;
default:
if (!calibrate(leftPupil.x, leftPupil.y, gaze.x, lcx, errorMsg))
return false;
if (!calibrate(leftPupil.x, leftPupil.y, gaze.y, lcy, errorMsg))
return false;
if (!calibrate(rightPupil.x, rightPupil.y, gaze.x, rcx, errorMsg))
return false;
if (!calibrate(rightPupil.x, rightPupil.y, gaze.y, rcy, errorMsg))
return false;
break;
switch(inputType) {
case BINOCULAR:
return binocularCalibrated;
case BINOCULAR_MEAN_POR:
return monoLeftCalibrated && monoRightCalibrated;
case MONO_LEFT:
return monoLeftCalibrated;
case MONO_RIGHT:
return monoRightCalibrated;
}
calibrationInputType = inputType;
......@@ -115,7 +113,7 @@ Point3f PolyFit::estimateGaze(const CollectionTuple &tuple, const InputType inpu
estimate3D.x = estimate.x;
estimate3D.y = estimate.y;
if (estimate.x < 0 || estimate.x >= depthMap.cols || estimate.y < 0 || estimate.y >= depthMap.rows)
estimate3D.z = depthMap.at<float>(0, 0);
estimate3D.z = -1;
else
estimate3D.z = depthMap.at<float>(estimate.y, estimate.x);
......@@ -125,11 +123,11 @@ Point3f PolyFit::estimateGaze(const CollectionTuple &tuple, const InputType inpu
Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType inputType)
{
Point2f estimate(-1, -1);
if (inputType != calibrationInputType)
return estimate;
switch (calibrationInputType) {
switch (inputType) {
case BINOCULAR:
if (!binocularCalibrated)
return estimate;
estimate.x = evaluate(
(leftPupil.x + rightPupil.x)/2.0,
(leftPupil.y + rightPupil.y)/2.0,
......@@ -141,6 +139,8 @@ Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType input
break;
case BINOCULAR_MEAN_POR:
if (!monoLeftCalibrated || !monoRightCalibrated)
return estimate;
estimate.x = evaluate(
leftPupil.x,
leftPupil.y,
......@@ -162,6 +162,8 @@ Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType input
break;
case MONO_LEFT:
if (!monoLeftCalibrated)
return estimate;
estimate.x = evaluate(
leftPupil.x,
leftPupil.y,
......@@ -173,6 +175,8 @@ Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType input
break;
case MONO_RIGHT:
if (!monoRightCalibrated)
return estimate;
estimate.x = evaluate(
rightPupil.x,
rightPupil.y,
......@@ -187,26 +191,6 @@ Point2f PolyFit::evaluate(Point2f leftPupil, Point2f rightPupil, InputType input
break;
}
if (calibrationInputType == BINOCULAR_MEAN_POR) {
Point2f le, re;
le.x = evaluate(
leftPupil.x,
leftPupil.y,
lcx);
le.y = evaluate(
leftPupil.x,
leftPupil.y,
lcy);
re.x = evaluate(
rightPupil.x,
rightPupil.y,
rcx);
re.y = evaluate(
rightPupil.x,
rightPupil.y,
rcy);
}
return estimate;
}
......
......@@ -52,6 +52,9 @@ private:
cv::Mat depthMap;
unsigned int unknowns;
InputType calibrationInputType;
bool binocularCalibrated;
bool monoLeftCalibrated;
bool monoRightCalibrated;
bool calibrate(cv::Mat &x, cv::Mat &y, cv::Mat &z, cv::Mat1d &c, QString &errorMsg);
float evaluate(float x, float y, cv::Mat1d &c);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment