GazeEstimation.cpp 27.1 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2
#include "GazeEstimation.h"

Thiago Santini's avatar
Thiago Santini committed
3 4
// TODO: This whole class need to be refactored and simplified

Thiago Santini's avatar
Thiago Santini committed
5 6 7
using namespace std;
using namespace cv;

8 9 10 11 12 13 14
GazeEstimation::GazeEstimation(QObject* parent)
    : QObject(parent)
    , calibrated(false)
    , isCalibrating(false)
    , gazeEstimationMethod(nullptr)
    , lastOverlayIdx(0)
    , settings(nullptr)
Thiago Santini's avatar
Thiago Santini committed
15
{
Thiago Santini's avatar
Thiago Santini committed
16 17 18

    availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY));
    availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY));
19 20 21 22
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XXYY));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXX_YYY));
Thiago Santini's avatar
Thiago Santini committed
23 24 25

    availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY));
    availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY));
26 27 28 29
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XXYY));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXX_YYY));
Thiago Santini's avatar
Thiago Santini committed
30

31
    availableGazeEstimationMethods.push_back(make_shared<Homography>());
32 33 34 35 36 37 38

    // Deactivated until the eye context gets integrated
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY, GazeVectorBinocularPolyFit::Mode::INSTANTANEOUS));
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY, GazeVectorBinocularPolyFit::Mode::INSTANTANEOUS));

    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY, GazeVectorBinocularPolyFit::Mode::TEMPORAL));
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY, GazeVectorBinocularPolyFit::Mode::TEMPORAL));
Thiago Santini's avatar
Thiago Santini committed
39
}
40

Thiago Santini's avatar
Thiago Santini committed
41 42 43 44 45 46 47 48 49 50 51
GazeEstimation::~GazeEstimation()
{
    availableGazeEstimationMethods.clear();
}

void GazeEstimation::addTuple(CollectionTuple tuple)
{
    collectedTuples.push_back(tuple);
}
void GazeEstimation::addTuples(std::vector<CollectionTuple> tuples)
{
52
    collectedTuples.insert(collectedTuples.end(), tuples.begin(), tuples.end());
Thiago Santini's avatar
Thiago Santini committed
53 54 55
}
void GazeEstimation::reset(CollectionTuple::TupleType type)
{
56
    for (size_t i = collectedTuples.size(); i-- > 0;)
Thiago Santini's avatar
Thiago Santini committed
57
        if (collectedTuples[i].tupleType == type)
58
            collectedTuples.erase(collectedTuples.begin() + i);
Thiago Santini's avatar
Thiago Santini committed
59 60
}

61
bool GazeEstimation::isPupilOutlineValid(const EyeData& cur)
Thiago Santini's avatar
Thiago Santini committed
62 63 64 65 66
{
    if (cur.pupil.size.width <= 0 || cur.pupil.size.height <= 0)
        return false;
    return true;
}
67
bool GazeEstimation::isPupilRatioValid(const EyeData& prev, const EyeData& cur)
Thiago Santini's avatar
Thiago Santini committed
68 69 70 71 72 73 74 75 76 77
{
    // TODO: parametrize me
    int temporalThresholdMs = 500;
    double ratioThreshold = 0.75;

    // too far away in time, disconsider
    if (cur.timestamp - prev.timestamp > temporalThresholdMs)
        return true;

    std::pair<double, double> values = std::minmax(
78 79
        cur.pupil.size.area(),
        prev.pupil.size.area());
Thiago Santini's avatar
Thiago Santini committed
80 81 82 83 84 85

    if (values.second == 0)
        return true;

    return (values.first / values.second < ratioThreshold) ? false : true;
}
86
void GazeEstimation::findPupilPositionOutliers(const Mat& pupil, Mat& mask)
Thiago Santini's avatar
Thiago Santini committed
87 88 89 90 91 92
{
    unsigned int outliers = 0;
    double sigmaThreshold = 2.7;
    while (true) {
        Scalar mu, s;
        meanStdDev(pupil, mu, s);
93
        for (int i = 0; i < pupil.rows; i++) {
Thiago Santini's avatar
Thiago Santini committed
94 95
            if (mask.at<uchar>(i) == 0)
                continue;
96 97 98 99
            if (pupil.at<Vec2f>(i)[0] < mu[0] - sigmaThreshold * s[0]
                || pupil.at<Vec2f>(i)[0] > mu[0] + sigmaThreshold * s[0]
                || pupil.at<Vec2f>(i)[1] < mu[1] - sigmaThreshold * s[1]
                || pupil.at<Vec2f>(i)[1] > mu[1] + sigmaThreshold * s[1]) {
Thiago Santini's avatar
Thiago Santini committed
100 101 102 103 104 105 106 107 108 109 110 111
                outliers++;
                mask.at<uchar>(i) = 0;
            }
        }
        if (outliers <= 0)
            break;
        outliers = 0;
    }
}
void GazeEstimation::detectOutliers()
{
    // reset outlier information
112
    for (unsigned int i = 0; i < calibrationTuples.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
113 114 115
        calibrationTuples[i]->outlierDesc = CollectionTuple::OD_INLIER;

    // Check for input type and pupil validity
116 117
    CollectionTuple* t;
    for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
118 119
        t = calibrationTuples[i];
        switch (cfg.inputType) {
120 121 122 123 124 125 126 127 128 129 130 131
        case GazeEstimationMethod::BINOCULAR:
            if (!t->lEye.validPupil || !t->rEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
        case GazeEstimationMethod::MONO_LEFT:
            if (!t->lEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
        case GazeEstimationMethod::MONO_RIGHT:
            if (!t->rEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
Thiago Santini's avatar
Thiago Santini committed
132 133 134 135 136 137 138
        }
    }

    if (!cfg.removeOutliers)
        return;

    if (cfg.pupilRatioOutliers) {
139 140 141
        for (unsigned int i = 1; i < calibrationTuples.size(); i++) {
            CollectionTuple* prev = calibrationTuples[i - 1];
            CollectionTuple* cur = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
142 143 144 145 146 147 148
            if (prev->isOutlier() || cur->isOutlier())
                continue;

            bool validLeft = isPupilRatioValid(prev->lEye, cur->lEye);
            bool validRight = isPupilRatioValid(prev->rEye, cur->rEye);

            switch (cfg.inputType) {
149 150 151 152 153 154 155 156 157 158 159 160
            case GazeEstimationMethod::BINOCULAR:
                if (!validLeft || !validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
            case GazeEstimationMethod::MONO_LEFT:
                if (!validLeft)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
            case GazeEstimationMethod::MONO_RIGHT:
                if (!validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
Thiago Santini's avatar
Thiago Santini committed
161 162 163 164 165 166 167
            }
        }
    }

    if (cfg.pupilPositionOutliers) {
        Mat lp(0, 0, CV_32FC2);
        Mat rp(0, 0, CV_32FC2);
168 169 170
        Mat mask(1, (int)calibrationTuples.size(), CV_8U);
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            CollectionTuple* t = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
171 172 173 174 175
            lp.push_back(t->lEye.pupil.center);
            rp.push_back(t->rEye.pupil.center);
            mask.at<uchar>(i) = t->isOutlier() ? 0 : 1;
        }
        if (cfg.inputType != GazeEstimationMethod::MONO_RIGHT)
176
            findPupilPositionOutliers(lp, mask);
Thiago Santini's avatar
Thiago Santini committed
177
        if (cfg.inputType != GazeEstimationMethod::MONO_LEFT)
178 179 180
            findPupilPositionOutliers(rp, mask);
        for (unsigned int i = 0; i < calibrationTuples.size(); i++)
            if (mask.at<uchar>(i) == (uchar)0)
Thiago Santini's avatar
Thiago Santini committed
181 182 183 184
                calibrationTuples[i]->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
    }

    if (cfg.pupilOutlineOutliers) {
185 186
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            CollectionTuple* cur = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
187 188 189 190 191 192 193
            if (cur->isOutlier())
                continue;

            bool validLeft = isPupilOutlineValid(cur->lEye);
            bool validRight = isPupilOutlineValid(cur->rEye);

            switch (cfg.inputType) {
194 195 196 197 198 199 200 201 202 203 204 205
            case GazeEstimationMethod::BINOCULAR:
                if (!validLeft || !validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
            case GazeEstimationMethod::MONO_LEFT:
                if (!validLeft)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
            case GazeEstimationMethod::MONO_RIGHT:
                if (!validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
Thiago Santini's avatar
Thiago Santini committed
206 207 208
            }
        }
    }
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229

    if (cfg.pupilDiameterOutliers) {
        vector<float> ld, rd;
        for (auto& cur : calibrationTuples) {
            if (cur->isOutlier())
                continue;
            ld.push_back(cur->lEye.pupil.majorAxis());
            rd.push_back(cur->rEye.pupil.majorAxis());
        }
        if (ld.size() > 0 && rd.size() > 0) {
            double lmed = median(ld);
            double rmed = median(rd);
            for (auto& cur : calibrationTuples) {
                if (cur->isOutlier())
                    continue;

                auto valid = [&](const double med, const EyeData ed, const float tolerance = 0.1) {
                    float ma = ed.pupil.majorAxis();
                    return ma < (1 + tolerance) * med && ma > (1 - tolerance);
                };

Thiago Santini's avatar
Thiago Santini committed
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
                bool validLeft = valid(lmed, cur->lEye);
                bool validRight = valid(rmed, cur->rEye);

                switch (cfg.inputType) {
                case GazeEstimationMethod::BINOCULAR:
                    if (validLeft || validRight)
                        continue;
                    break;
                case GazeEstimationMethod::MONO_LEFT:
                    if (validLeft)
                        continue;
                    break;
                case GazeEstimationMethod::MONO_RIGHT:
                    if (validRight)
                        continue;
                    break;
                }
247 248 249 250 251

                cur->outlierDesc = CollectionTuple::OD_PUPIL_DIAMETER;
            }
        }
    }
Thiago Santini's avatar
Thiago Santini committed
252 253 254 255
}

void GazeEstimation::calibrate()
{
256 257
    updateTemporalGazes(collectedTuples);

Thiago Santini's avatar
Thiago Santini committed
258 259 260 261 262 263
    calibrated = false;
    autoVisualizationTimer.invalidate();
    errorVectors.clear();
    interpolationHull.clear();
    evaluationRegions.clear();

264 265 266 267 268 269 270 271
    QString error = "";
    if (!gazeEstimationMethod) {
        error = "No gaze estimation method available.";
        error = "No calibration tuples available.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }
Thiago Santini's avatar
Thiago Santini committed
272 273 274

    calibrationTuples.clear();

275
    for (unsigned int i = 0; i < collectedTuples.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
276 277 278 279 280 281 282 283 284 285 286 287 288
        if (collectedTuples[i].isCalibration())
            calibrationTuples.push_back(&collectedTuples[i]);
    if (calibrationTuples.size() <= 0) {
        error = "No calibration tuples available.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }

    detectOutliers();

    if (cfg.autoEvaluation)
        selectEvaluationTuples(cfg.granularity, cfg.horizontalStride, cfg.verticalStride, cfg.rangeFactor);
289 290 291 292 293
    else {
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            calibrationTuples[i]->autoEval = CollectionTuple::AE_NO;
        }
    }
Thiago Santini's avatar
Thiago Santini committed
294 295

    vector<CollectionTuple> calibrationInliers;
Thiago Santini's avatar
Thiago Santini committed
296 297 298
    for (const auto& tuple : calibrationTuples)
        if (tuple->useForCalibration())
            calibrationInliers.push_back(*tuple);
Thiago Santini's avatar
Thiago Santini committed
299

300 301
    updateInterpolationHull(calibrationInliers);

Thiago Santini's avatar
Thiago Santini committed
302 303 304 305 306 307 308 309
    qInfo() << "Using" << calibrationInliers.size() << "/" << calibrationTuples.size() << "calibration tuples.";
    if (calibrationInliers.size() <= 0) {
        error = "No calibration tuples remaining.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }

310
    calibrated = gazeEstimationMethod->calibrate(calibrationInliers, error);
Thiago Santini's avatar
Thiago Santini committed
311 312 313 314 315 316

    if (!calibrated) {
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }
317 318 319 320 321 322
    evaluate();

    if (centralHullCoverage < cfg.minCentralAreaCoverage)
        error.append("Calibration didn't cover enough of the central area. ");
    if (peripheralHullCoverage < cfg.minPeriphericAreaCoverage)
        error.append("Calibration didn't cover enough of the peripheric area. ");
323
    if (cfg.maxReprojectionError > 0 && 100 * meanEvaluationError > cfg.maxReprojectionError)
324 325 326 327 328 329 330
        error.append("Reprojection error is too high. ");
    if (!error.isEmpty()) {
        qInfo() << error;
        emit calibrationFinished(false, error);
        calibrated = false;
        return;
    }
331

Thiago Santini's avatar
Thiago Santini committed
332 333 334 335
    autoVisualizationTimer.restart();
    emit calibrationFinished(true, "");
}

336
GazeEstimate GazeEstimation::getGazeEstimate(const DataTuple& dataTuple)
Thiago Santini's avatar
Thiago Santini committed
337
{
338 339
    GazeEstimate gazeEstimate;
    if (calibrated && gazeEstimationMethod) {
340
        GazeEstimate left, right, binocular;
341
        gazeEstimationMethod->estimate(dataTuple, left, right, binocular);
342

343
        switch (cfg.inputType) {
344
        case GazeEstimationMethod::BINOCULAR:
345
            gazeEstimate = binocular;
346 347
            break;
        case GazeEstimationMethod::MONO_LEFT:
348
            gazeEstimate = left;
349 350
            break;
        case GazeEstimationMethod::MONO_RIGHT:
351
            gazeEstimate = right;
352
            break;
353
        }
Thiago Santini's avatar
Thiago Santini committed
354
    }
355 356
    return gazeEstimate;
}
Thiago Santini's avatar
Thiago Santini committed
357

358 359 360
void GazeEstimation::estimate(DataTuple dataTuple)
{
    dataTuple.field.gazeEstimate = getGazeEstimate(dataTuple);
Thiago Santini's avatar
Thiago Santini committed
361 362 363 364
    drawGazeEstimationInfo(dataTuple);
    emit gazeEstimationDone(dataTuple);
}

365
void GazeEstimation::printAccuracyInfo(const cv::Mat& errors, const QString& which, const double& diagonal, float& mu, float& sigma)
366 367 368
{
    Scalar m, std;
    meanStdDev(errors, m, std);
369
    qInfo() << which.toLatin1().data() << "Error ( N =" << errors.rows << "):";
370 371
    //qInfo() << QString("m = %1 ( std = %2 ) pixels."
    //    ).arg(m[0], 6, 'f', 2
372 373 374 375
    //    ).arg(std[0], 6, 'f', 2).toLatin1().data();
    mu = m[0] / diagonal;
    sigma = std[0] / diagonal;
    qInfo() << QString("m = %1 ( std = %2 ) % of the image diagonal.").arg(100 * mu, 6, 'f', 2).arg(100 * sigma, 6, 'f', 2).toLatin1().data();
376 377
}

Thiago Santini's avatar
Thiago Santini committed
378 379 380 381 382 383
void GazeEstimation::evaluate()
{
    if (!calibrated)
        return;

    evaluationTuples.clear();
384
    for (unsigned int i = 0; i < collectedTuples.size(); i++)
385
        if (collectedTuples[i].isEvaluation() || (cfg.autoEvaluation && collectedTuples[i].isAutoEval())) {
386
            if (getGazeEstimate(collectedTuples[i]).valid)
387 388
                evaluationTuples.push_back(&collectedTuples[i]);
        }
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425

    centralHullCoverage = 0;
    peripheralHullCoverage = 0;
    if (!interpolationHull.empty()) {
        float delta = 0.25;
        Size size(calibrationTuples[0]->field.width, calibrationTuples[0]->field.height);
        Mat centralRegion = Mat::zeros(size, CV_8U);
        rectangle(centralRegion, Point(delta * size.width, delta * size.height), Point((1 - delta) * size.width, (1 - delta) * size.height), Scalar(255), -1);
        Mat peripheralRegion = 255 - centralRegion;

        Mat hullRegion = Mat::zeros(size, CV_8U);
        vector<vector<Point>> contours;
        contours.push_back(interpolationHull);
        drawContours(hullRegion, contours, 0, Scalar(255), -1);

        vector<Point> nonZero;
        findNonZero(centralRegion, nonZero);
        float centralPixels = nonZero.size();
        findNonZero(peripheralRegion, nonZero);
        float peripheralPixels = nonZero.size();

        Mat intersection;
        bitwise_and(hullRegion, centralRegion, intersection);
        findNonZero(intersection, nonZero);
        centralHullCoverage = nonZero.size() / centralPixels;

        bitwise_and(hullRegion, peripheralRegion, intersection);
        findNonZero(intersection, nonZero);
        peripheralHullCoverage = nonZero.size() / peripheralPixels;

        qInfo() << QString("Central Interpolation Coverage: %1 %").arg(100 * centralHullCoverage, 0, 'f', 2).toLatin1().data();
        qInfo() << QString("Peripheral Interpolation Coverage: %1 %").arg(100 * peripheralHullCoverage, 0, 'f', 2).toLatin1().data();
    }

    meanEvaluationError = 0;
    stdEvaluationError = 0;
    evaluationRegionCoverage = 1;
Thiago Santini's avatar
Thiago Santini committed
426 427

    if (evaluationTuples.size() <= 0 || calibrationTuples.size() <= 0)
428
        return;
Thiago Santini's avatar
Thiago Santini committed
429

430
    // evaluate for known points
Thiago Santini's avatar
Thiago Santini committed
431
    errorVectors.clear();
432
    Mat errors;
433
    for (unsigned int i = 0; i < evaluationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
434
        Point3f gt = evaluationTuples[i]->field.collectionMarker.center;
435
        Point3f gaze = to3D(getGazeEstimate(*evaluationTuples[i]).gp);
Thiago Santini's avatar
Thiago Santini committed
436
        errorVectors.push_back(ErrorVector(gt, gaze));
437
        errors.push_back(errorVectors.back().magnitude());
Thiago Santini's avatar
Thiago Santini committed
438 439
    }

440 441
    double imDiagonal = sqrt(pow(evaluationTuples[0]->field.width, 2) + pow(evaluationTuples[0]->field.height, 2));
    printAccuracyInfo(errors, "Gaze Evaluation", imDiagonal, meanEvaluationError, stdEvaluationError);
442 443

    if (cfg.autoEvaluation) {
444
        double evaluationRegionsCount = pow(1 + 2 * cfg.granularity, 2);
445
        double evaluationRegionsSelected = 0;
446 447
        for (unsigned int i = 0; i < collectedTuples.size(); i++)
            if (collectedTuples[i].isAutoEval())
448
                evaluationRegionsSelected++;
449 450 451
        evaluationRegionCoverage = evaluationRegionsSelected / evaluationRegionsCount;
        qInfo() << QString("Auto Evaluation Region Coverage: %1 %").arg(100 * evaluationRegionCoverage, 0, 'f', 2).toLatin1().data();
    }
Thiago Santini's avatar
Thiago Santini committed
452 453 454 455 456
}

/*
 * File loading / saving
 */
457 458
template <typename T>
static void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key, T& data)
Thiago Santini's avatar
Thiago Santini committed
459 460 461 462 463 464
{
    if (idxMap.count(key) == 0)
        return;

    data = QVariant(tokens[idxMap[key]]).value<T>();
}
465
template <>
466
inline void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key, vector<Marker>& markers)
467 468 469 470 471 472 473 474 475 476 477
{
    if (idxMap.count(key) == 0)
        return;

    QString markersStr = tokens[idxMap[key]];
    for (auto& markerStr : markersStr.split(Token::MarkerEnd)) {
        Marker tmp;
        if (tmp.parse(markerStr))
            markers.push_back(std::move(tmp));
    }
}
Thiago Santini's avatar
Thiago Santini committed
478 479 480
void GazeEstimation::loadTuplesFromFile(CollectionTuple::TupleType tupleType, QString fileName)
{
    vector<CollectionTuple> tuples;
481 482 483 484 485 486 487
    TSVReader tsv;
    if (!tsv.open(fileName))
        return;
    for (int i = 0; i < tsv.size(); i++) {
        CollectionTuple tuple = CollectionTuple(DataTuple(tsv, i));
        tuple.tupleType = tupleType;
        tuples.emplace_back(tuple);
Thiago Santini's avatar
Thiago Santini committed
488 489 490 491 492 493 494
    }
    addTuples(tuples);
    calibrate();
}

void GazeEstimation::saveTuplesToFile(CollectionTuple::TupleType tupleType, QString fileName)
{
495 496 497 498 499 500 501 502 503 504
    vector<CollectionTuple*> pertinentTuples;
    for (unsigned int i = 0; i < collectedTuples.size(); i++) {
        //if ( tupleType == CollectionTuple::CALIBRATION)
        //	if (collectedTuples[i].isEvaluation())
        //		continue;

        if (tupleType == CollectionTuple::EVALUATION) {
            if (collectedTuples[i].isCalibration())
                continue;
        }
Thiago Santini's avatar
Thiago Santini committed
505

506 507
        pertinentTuples.push_back(&collectedTuples[i]);
    }
Thiago Santini's avatar
Thiago Santini committed
508

509
    saveTuplesToFile(pertinentTuples, fileName, QIODevice::WriteOnly);
Thiago Santini's avatar
Thiago Santini committed
510 511
}

512
void GazeEstimation::saveTuplesToFile(const std::vector<CollectionTuple*>& tuples, QString fileName, QFlags<QIODevice::OpenModeFlag> flags)
Thiago Santini's avatar
Thiago Santini committed
513 514 515 516 517
{
    if (fileName.isEmpty() || tuples.size() == 0)
        return;

    QString buffer;
518 519 520
    buffer.append(tuples[0]->header() + Token::Newline);
    for (unsigned int i = 0; i < tuples.size(); i++)
        buffer.append(tuples[i]->toQString() + Token::Newline);
Thiago Santini's avatar
Thiago Santini committed
521 522 523 524 525 526 527 528 529 530 531 532

    QFile f(fileName);
    f.open(flags);
    QTextStream s(&f);
    if (s.status() == QTextStream::Ok)
        s << buffer.toStdString().c_str();
    f.close();
}

void GazeEstimation::updateConfig()
{
    cfg.load(settings);
533
    for (int i = 0; i < availableGazeEstimationMethods.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
534
        QString name = availableGazeEstimationMethods[i]->description().c_str();
Thiago Santini's avatar
Thiago Santini committed
535
        if (cfg.gazeEstimationMethod == name) {
Thiago Santini's avatar
Thiago Santini committed
536
            gazeEstimationMethod = availableGazeEstimationMethods[i];
Thiago Santini's avatar
Thiago Santini committed
537 538 539 540 541 542 543 544 545
            break;
        }
    }
    if (!gazeEstimationMethod) {
        qWarning() << "Unknown gaze estimation method:" << cfg.gazeEstimationMethod;
        if (availableGazeEstimationMethods.size() > 0) {
            gazeEstimationMethod = availableGazeEstimationMethods.front();
            qWarning() << "Defaulting to" << gazeEstimationMethod->description().c_str();
        }
Thiago Santini's avatar
Thiago Santini committed
546 547 548 549
    }
    calibrate();
}

550
void GazeEstimation::selectEvaluationTuples(const int g, const double dx, const double dy, const double rf)
Thiago Santini's avatar
Thiago Santini committed
551 552 553 554 555 556 557 558 559 560
{
    vector<CollectionTuple*> evaluationCandidates;
    vector<Point> points;
    CollectionTuple* t;

    evaluationRegions.clear();

    /*
     * collect non-outlier candidates
     */
561
    for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
562 563 564 565 566
        t = calibrationTuples[i];
        t->autoEval = CollectionTuple::AE_NO;
        if (t->isOutlier())
            continue;
        evaluationCandidates.push_back(t);
567
        points.push_back(to2D(t->field.collectionMarker.center));
Thiago Santini's avatar
Thiago Santini committed
568 569 570 571
    }

    if (evaluationCandidates.size() <= 0)
        return;
572
    t = evaluationCandidates[0];
Thiago Santini's avatar
Thiago Santini committed
573 574 575 576

    /*
     *  exclude points in the hull so we don't lose interpolation range
     */
Thiago Santini's avatar
Thiago Santini committed
577
    const float minDistFromHullPx = 3;
Thiago Santini's avatar
Thiago Santini committed
578 579 580 581
    convexHull(points, interpolationHull);

//#define DBG_AUTOEVAL
#ifdef DBG_AUTOEVAL
582 583 584 585 586 587 588
    const Point* ep[1] = { &interpolationHull[0] };
    int epn = (int)interpolationHull.size();
    Mat dbg = Mat::zeros(evaluationCandidates[0]->field.height, evaluationCandidates[0]->field.width, CV_8UC3);
    fillPoly(dbg, ep, &epn, 1, Scalar(255, 255, 255));
    for (size_t i = evaluationCandidates.size(); i-- > 0;) {
        Point gaze = to2D(evaluationCandidates[i]->field.collectionMarker.center);
        circle(dbg, gaze, 5, Scalar(0, 255, 255), -1);
Thiago Santini's avatar
Thiago Santini committed
589 590 591
    }
#endif

592 593 594
    for (size_t i = evaluationCandidates.size(); i-- > 0;) {
        Point gaze = to2D(evaluationCandidates[i]->field.collectionMarker.center);
        for (unsigned int j = 0; j < interpolationHull.size(); j++) {
Thiago Santini's avatar
Thiago Santini committed
595
            if (ED(gaze, interpolationHull[j]) <= minDistFromHullPx) {
Thiago Santini's avatar
Thiago Santini committed
596
#ifdef DBG_AUTOEVAL
597
                circle(dbg, gaze, 5, Scalar(255, 0, 0), -1);
Thiago Santini's avatar
Thiago Santini committed
598
#endif
599
                evaluationCandidates.erase(evaluationCandidates.begin() + i);
Thiago Santini's avatar
Thiago Santini committed
600 601 602 603 604 605 606 607
                break;
            }
        }
    }

    /*
     * Select points that minimize distance to evaluation region center for evaluation
     */
608 609 610 611 612
    Point center(t->field.width / 2, t->field.height / 2);
    int dxPx = dx * t->field.width;
    int dyPx = dy * t->field.height;
    int w = rf * dxPx;
    int h = rf * dyPx;
Thiago Santini's avatar
Thiago Santini committed
613 614 615
    vector<Point> evaluationPoints;
    for (int x = -g; x <= g; x++) {
        for (int y = -g; y <= g; y++) {
616 617
            EvaluationRegion er(center.x + x * dxPx, center.y + y * dyPx, w, h);
            for (int i = 0; i < evaluationCandidates.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
                er.eval(evaluationCandidates[i]);
            if (er.selected) {
                er.selected->autoEval = CollectionTuple::AE_YES;
                evaluationPoints.push_back(to2D(er.selected->field.collectionMarker.center));
            }
            evaluationRegions.push_back(er);
#ifdef DBG_AUTOEVAL
            er.draw(dbg);
#endif
        }
    }

    /*
     * Mark calibration points that could bias the auto evaluation
     */
Thiago Santini's avatar
Thiago Santini committed
633
    const float minDistFromEvaluationPx = 5;
634
    for (unsigned int i = 0; i < evaluationPoints.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
635
        Point ep = evaluationPoints[i];
636
        for (unsigned int j = 0; j < evaluationCandidates.size(); j++) {
Thiago Santini's avatar
Thiago Santini committed
637 638 639
            t = evaluationCandidates[j];
            if (t->autoEval == CollectionTuple::AE_NO) {
                Point cp = to2D(t->field.collectionMarker.center);
Thiago Santini's avatar
Thiago Santini committed
640
                if (ED(ep, cp) < minDistFromEvaluationPx) {
Thiago Santini's avatar
Thiago Santini committed
641 642
                    t->autoEval = CollectionTuple::AE_BIASED;
#ifdef DBG_AUTOEVAL
643
                    circle(dbg, cp, 2, Scalar(255, 255, 0), -1);
Thiago Santini's avatar
Thiago Santini committed
644 645 646 647 648 649 650 651 652 653 654
#endif
                }
            }
        }
    }

#ifdef DBG_AUTOEVAL
    imshow("dbg", dbg);
#endif
}

655
void GazeEstimation::drawGazeEstimationInfo(DataTuple& dataTuple)
Thiago Santini's avatar
Thiago Santini committed
656 657 658 659 660 661 662 663 664 665 666 667 668
{
    dataTuple.showGazeEstimationVisualization = false;

    // TODO: improve this? maybe draw on an overlay canvas that is only updated
    // when something changes. Then we overlay the canvas on a copy of the image
    if (dataTuple.field.input.empty() || !cfg.visualize)
        return;

    bool shouldDisplay = false;
    if (isCalibrating)
        shouldDisplay = true;
    else {
        if (autoVisualizationTimer.isValid()) {
669
            if (autoVisualizationTimer.elapsed() < cfg.visualizationTimeS * 1e3)
Thiago Santini's avatar
Thiago Santini committed
670 671 672 673 674
                shouldDisplay = true;
            else
                autoVisualizationTimer.invalidate();
        }
    }
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
    if (!shouldDisplay) {
        lastOverlayIdx = 0;
        return;
    }

    dataTuple.showGazeEstimationVisualization = true;
    dataTuple.gazeEstimationVisualization = vis;

    // avoid drawing every single frame
    //static Timestamp lastGazeEstimationVisualizationTimestamp = 0;
    //Timestamp current = gTimer.elapsed();
    //bool shouldDraw = current - lastGazeEstimationVisualizationTimestamp > 40;
    //if (!shouldDraw)
    //	return;
    //lastGazeEstimationVisualizationTimestamp = current;

    vis = dataTuple.field.input.clone();
    int r = max<int>(1, 0.003125 * max<int>(vis.rows, vis.cols));

    if (isCalibrating) {

        if (lastOverlayIdx > collectedTuples.size()) // sample removed, restart
            lastOverlayIdx = 0;

        if (lastOverlayIdx == 0)
            overlay = Mat::zeros(vis.rows, vis.cols, CV_8UC3);

        for (; lastOverlayIdx < collectedTuples.size(); lastOverlayIdx++) {
            circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r, CV_ALMOST_BLACK, -1);
            if (collectedTuples[lastOverlayIdx].isCalibration())
                circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r + 1, CV_GREEN, 0.5 * r);
            else
                circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r + 1, CV_CYAN, 0.5 * r);
        }

    } else {
        // Calibration finished; overlay results and restart
        overlay = Mat::zeros(vis.rows, vis.cols, CV_8UC3);
        lastOverlayIdx = 0;

        if (!interpolationHull.empty()) {
            vector<vector<Point>> contours;
Thiago Santini's avatar
Thiago Santini committed
717
            contours.push_back(interpolationHull);
718
            drawContours(overlay, contours, 0, CV_GREEN, r);
Thiago Santini's avatar
Thiago Santini committed
719 720
        }

721 722
        for (size_t i = 0; i < errorVectors.size(); i++)
            errorVectors[i].draw(overlay, 2, CV_RED);
Thiago Santini's avatar
Thiago Santini committed
723 724

        if (cfg.autoEvaluation)
725 726
            for (size_t i = 0; i < evaluationRegions.size(); i++)
                evaluationRegions[i].draw(overlay, r);
Thiago Santini's avatar
Thiago Santini committed
727

728 729 730 731 732 733 734
        for (size_t i = 0; i < collectedTuples.size(); i++) {
            if (!collectedTuples[i].isCalibration()) {
                circle(overlay, to2D(collectedTuples[i].field.collectionMarker.center), r, CV_BLACK, -1);
                circle(overlay, to2D(collectedTuples[i].field.collectionMarker.center), r + 1, CV_CYAN, 0.5 * r);
            }
        }
    }
Thiago Santini's avatar
Thiago Santini committed
735

736 737 738
    // Overlay on visualization image; notice we use CV_ALMOST_BLACK instead of
    // CV_BLACK so we don't need to create an additional mask :-)
    overlay.copyTo(vis, overlay);
Thiago Santini's avatar
Thiago Santini committed
739 740 741 742 743 744 745
}

void GazeEstimation::setCalibrating(bool v)
{
    isCalibrating = v;
}

746
void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>& tuples)
747 748 749 750 751
{
    if (tuples.size() <= 0)
        return;

    vector<Point> points;
752 753
    for (unsigned int i = 0; i < tuples.size(); i++)
        points.push_back(to2D(tuples[i].field.collectionMarker.center));
754 755 756

    convexHull(points, interpolationHull);
}
757 758 759

void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples)
{
Thiago Santini's avatar
Thiago Santini committed
760
    (void)tuples;
761
}