GazeEstimation.cpp 27.1 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2
#include "GazeEstimation.h"

Thiago Santini's avatar
Thiago Santini committed
3 4
// TODO: This whole class need to be refactored and simplified

Thiago Santini's avatar
Thiago Santini committed
5 6 7
using namespace std;
using namespace cv;

8 9 10 11 12 13 14
GazeEstimation::GazeEstimation(QObject* parent)
    : QObject(parent)
    , calibrated(false)
    , isCalibrating(false)
    , gazeEstimationMethod(nullptr)
    , lastOverlayIdx(0)
    , settings(nullptr)
Thiago Santini's avatar
Thiago Santini committed
15
{
Thiago Santini's avatar
Thiago Santini committed
16 17 18 19 20 21
    availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY));
    availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XXYY));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX));
    //availableGazeEstimationMethods.push_back(make_shared<BinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXX_YYY));
Thiago Santini's avatar
Thiago Santini committed
22 23 24

    availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY));
    availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY));
25 26 27 28
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XXYY));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX));
    //availableGazeEstimationMethods.push_back(make_shared<PolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXX_YYY));
Thiago Santini's avatar
Thiago Santini committed
29

30
    availableGazeEstimationMethods.push_back(make_shared<Homography>());
31 32 33 34 35 36 37

    // Deactivated until the eye context gets integrated
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY, GazeVectorBinocularPolyFit::Mode::INSTANTANEOUS));
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY, GazeVectorBinocularPolyFit::Mode::INSTANTANEOUS));

    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY_XX_YY_XYY_YXX_XXYY, GazeVectorBinocularPolyFit::Mode::TEMPORAL));
    //availableGazeEstimationMethods.push_back(make_shared<GazeVectorBinocularPolyFit>(PolyFit::POLY_X_Y_XY, GazeVectorBinocularPolyFit::Mode::TEMPORAL));
Thiago Santini's avatar
Thiago Santini committed
38
}
39

Thiago Santini's avatar
Thiago Santini committed
40 41 42 43 44 45 46 47 48 49 50
GazeEstimation::~GazeEstimation()
{
    availableGazeEstimationMethods.clear();
}

void GazeEstimation::addTuple(CollectionTuple tuple)
{
    collectedTuples.push_back(tuple);
}
void GazeEstimation::addTuples(std::vector<CollectionTuple> tuples)
{
51
    collectedTuples.insert(collectedTuples.end(), tuples.begin(), tuples.end());
Thiago Santini's avatar
Thiago Santini committed
52 53 54
}
void GazeEstimation::reset(CollectionTuple::TupleType type)
{
55
    for (size_t i = collectedTuples.size(); i-- > 0;)
Thiago Santini's avatar
Thiago Santini committed
56
        if (collectedTuples[i].tupleType == type)
57
            collectedTuples.erase(collectedTuples.begin() + i);
Thiago Santini's avatar
Thiago Santini committed
58 59
}

60
bool GazeEstimation::isPupilOutlineValid(const EyeData& cur)
Thiago Santini's avatar
Thiago Santini committed
61 62 63 64 65
{
    if (cur.pupil.size.width <= 0 || cur.pupil.size.height <= 0)
        return false;
    return true;
}
66
bool GazeEstimation::isPupilRatioValid(const EyeData& prev, const EyeData& cur)
Thiago Santini's avatar
Thiago Santini committed
67 68 69 70 71 72 73 74 75 76
{
    // TODO: parametrize me
    int temporalThresholdMs = 500;
    double ratioThreshold = 0.75;

    // too far away in time, disconsider
    if (cur.timestamp - prev.timestamp > temporalThresholdMs)
        return true;

    std::pair<double, double> values = std::minmax(
77 78
        cur.pupil.size.area(),
        prev.pupil.size.area());
Thiago Santini's avatar
Thiago Santini committed
79 80 81 82 83 84

    if (values.second == 0)
        return true;

    return (values.first / values.second < ratioThreshold) ? false : true;
}
85
void GazeEstimation::findPupilPositionOutliers(const Mat& pupil, Mat& mask)
Thiago Santini's avatar
Thiago Santini committed
86 87 88 89 90 91
{
    unsigned int outliers = 0;
    double sigmaThreshold = 2.7;
    while (true) {
        Scalar mu, s;
        meanStdDev(pupil, mu, s);
92
        for (int i = 0; i < pupil.rows; i++) {
Thiago Santini's avatar
Thiago Santini committed
93 94
            if (mask.at<uchar>(i) == 0)
                continue;
95 96 97 98
            if (pupil.at<Vec2f>(i)[0] < mu[0] - sigmaThreshold * s[0]
                || pupil.at<Vec2f>(i)[0] > mu[0] + sigmaThreshold * s[0]
                || pupil.at<Vec2f>(i)[1] < mu[1] - sigmaThreshold * s[1]
                || pupil.at<Vec2f>(i)[1] > mu[1] + sigmaThreshold * s[1]) {
Thiago Santini's avatar
Thiago Santini committed
99 100 101 102 103 104 105 106 107 108 109 110
                outliers++;
                mask.at<uchar>(i) = 0;
            }
        }
        if (outliers <= 0)
            break;
        outliers = 0;
    }
}
void GazeEstimation::detectOutliers()
{
    // reset outlier information
111
    for (unsigned int i = 0; i < calibrationTuples.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
112 113 114
        calibrationTuples[i]->outlierDesc = CollectionTuple::OD_INLIER;

    // Check for input type and pupil validity
115 116
    CollectionTuple* t;
    for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
117 118
        t = calibrationTuples[i];
        switch (cfg.inputType) {
119 120 121 122 123 124 125 126 127 128 129 130
        case GazeEstimationMethod::BINOCULAR:
            if (!t->lEye.validPupil || !t->rEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
        case GazeEstimationMethod::MONO_LEFT:
            if (!t->lEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
        case GazeEstimationMethod::MONO_RIGHT:
            if (!t->rEye.validPupil)
                t->outlierDesc = CollectionTuple::OD_MISSING_INPUT;
            break;
Thiago Santini's avatar
Thiago Santini committed
131 132 133 134 135 136 137
        }
    }

    if (!cfg.removeOutliers)
        return;

    if (cfg.pupilRatioOutliers) {
138 139 140
        for (unsigned int i = 1; i < calibrationTuples.size(); i++) {
            CollectionTuple* prev = calibrationTuples[i - 1];
            CollectionTuple* cur = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
141 142 143 144 145 146 147
            if (prev->isOutlier() || cur->isOutlier())
                continue;

            bool validLeft = isPupilRatioValid(prev->lEye, cur->lEye);
            bool validRight = isPupilRatioValid(prev->rEye, cur->rEye);

            switch (cfg.inputType) {
148 149 150 151 152 153 154 155 156 157 158 159
            case GazeEstimationMethod::BINOCULAR:
                if (!validLeft || !validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
            case GazeEstimationMethod::MONO_LEFT:
                if (!validLeft)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
            case GazeEstimationMethod::MONO_RIGHT:
                if (!validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_RATIO;
                break;
Thiago Santini's avatar
Thiago Santini committed
160 161 162 163 164 165 166
            }
        }
    }

    if (cfg.pupilPositionOutliers) {
        Mat lp(0, 0, CV_32FC2);
        Mat rp(0, 0, CV_32FC2);
167 168 169
        Mat mask(1, (int)calibrationTuples.size(), CV_8U);
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            CollectionTuple* t = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
170 171 172 173 174
            lp.push_back(t->lEye.pupil.center);
            rp.push_back(t->rEye.pupil.center);
            mask.at<uchar>(i) = t->isOutlier() ? 0 : 1;
        }
        if (cfg.inputType != GazeEstimationMethod::MONO_RIGHT)
175
            findPupilPositionOutliers(lp, mask);
Thiago Santini's avatar
Thiago Santini committed
176
        if (cfg.inputType != GazeEstimationMethod::MONO_LEFT)
177 178 179
            findPupilPositionOutliers(rp, mask);
        for (unsigned int i = 0; i < calibrationTuples.size(); i++)
            if (mask.at<uchar>(i) == (uchar)0)
Thiago Santini's avatar
Thiago Santini committed
180 181 182 183
                calibrationTuples[i]->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
    }

    if (cfg.pupilOutlineOutliers) {
184 185
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            CollectionTuple* cur = calibrationTuples[i];
Thiago Santini's avatar
Thiago Santini committed
186 187 188 189 190 191 192
            if (cur->isOutlier())
                continue;

            bool validLeft = isPupilOutlineValid(cur->lEye);
            bool validRight = isPupilOutlineValid(cur->rEye);

            switch (cfg.inputType) {
193 194 195 196 197 198 199 200 201 202 203 204
            case GazeEstimationMethod::BINOCULAR:
                if (!validLeft || !validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
            case GazeEstimationMethod::MONO_LEFT:
                if (!validLeft)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
            case GazeEstimationMethod::MONO_RIGHT:
                if (!validRight)
                    cur->outlierDesc = CollectionTuple::OD_PUPIL_POSITION;
                break;
Thiago Santini's avatar
Thiago Santini committed
205 206 207
            }
        }
    }
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228

    if (cfg.pupilDiameterOutliers) {
        vector<float> ld, rd;
        for (auto& cur : calibrationTuples) {
            if (cur->isOutlier())
                continue;
            ld.push_back(cur->lEye.pupil.majorAxis());
            rd.push_back(cur->rEye.pupil.majorAxis());
        }
        if (ld.size() > 0 && rd.size() > 0) {
            double lmed = median(ld);
            double rmed = median(rd);
            for (auto& cur : calibrationTuples) {
                if (cur->isOutlier())
                    continue;

                auto valid = [&](const double med, const EyeData ed, const float tolerance = 0.1) {
                    float ma = ed.pupil.majorAxis();
                    return ma < (1 + tolerance) * med && ma > (1 - tolerance);
                };

Thiago Santini's avatar
Thiago Santini committed
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
                bool validLeft = valid(lmed, cur->lEye);
                bool validRight = valid(rmed, cur->rEye);

                switch (cfg.inputType) {
                case GazeEstimationMethod::BINOCULAR:
                    if (validLeft || validRight)
                        continue;
                    break;
                case GazeEstimationMethod::MONO_LEFT:
                    if (validLeft)
                        continue;
                    break;
                case GazeEstimationMethod::MONO_RIGHT:
                    if (validRight)
                        continue;
                    break;
                }
246 247 248 249 250

                cur->outlierDesc = CollectionTuple::OD_PUPIL_DIAMETER;
            }
        }
    }
Thiago Santini's avatar
Thiago Santini committed
251 252 253 254
}

void GazeEstimation::calibrate()
{
255 256
    updateTemporalGazes(collectedTuples);

Thiago Santini's avatar
Thiago Santini committed
257 258 259 260 261 262
    calibrated = false;
    autoVisualizationTimer.invalidate();
    errorVectors.clear();
    interpolationHull.clear();
    evaluationRegions.clear();

263 264 265 266 267 268 269 270
    QString error = "";
    if (!gazeEstimationMethod) {
        error = "No gaze estimation method available.";
        error = "No calibration tuples available.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }
Thiago Santini's avatar
Thiago Santini committed
271 272 273

    calibrationTuples.clear();

274
    for (unsigned int i = 0; i < collectedTuples.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
275 276 277 278 279 280 281 282 283 284 285 286 287
        if (collectedTuples[i].isCalibration())
            calibrationTuples.push_back(&collectedTuples[i]);
    if (calibrationTuples.size() <= 0) {
        error = "No calibration tuples available.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }

    detectOutliers();

    if (cfg.autoEvaluation)
        selectEvaluationTuples(cfg.granularity, cfg.horizontalStride, cfg.verticalStride, cfg.rangeFactor);
288 289 290 291 292
    else {
        for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
            calibrationTuples[i]->autoEval = CollectionTuple::AE_NO;
        }
    }
Thiago Santini's avatar
Thiago Santini committed
293 294

    vector<CollectionTuple> calibrationInliers;
Thiago Santini's avatar
Thiago Santini committed
295 296 297
    for (const auto& tuple : calibrationTuples)
        if (tuple->useForCalibration())
            calibrationInliers.push_back(*tuple);
Thiago Santini's avatar
Thiago Santini committed
298

299 300
    updateInterpolationHull(calibrationInliers);

Thiago Santini's avatar
Thiago Santini committed
301 302 303 304 305 306 307 308
    qInfo() << "Using" << calibrationInliers.size() << "/" << calibrationTuples.size() << "calibration tuples.";
    if (calibrationInliers.size() <= 0) {
        error = "No calibration tuples remaining.";
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }

309
    calibrated = gazeEstimationMethod->calibrate(calibrationInliers, error);
Thiago Santini's avatar
Thiago Santini committed
310 311 312 313 314 315

    if (!calibrated) {
        qInfo() << error;
        emit calibrationFinished(false, error);
        return;
    }
316 317 318 319 320 321
    evaluate();

    if (centralHullCoverage < cfg.minCentralAreaCoverage)
        error.append("Calibration didn't cover enough of the central area. ");
    if (peripheralHullCoverage < cfg.minPeriphericAreaCoverage)
        error.append("Calibration didn't cover enough of the peripheric area. ");
322
    if (cfg.maxReprojectionError > 0 && 100 * meanEvaluationError > cfg.maxReprojectionError)
323 324 325 326 327 328 329
        error.append("Reprojection error is too high. ");
    if (!error.isEmpty()) {
        qInfo() << error;
        emit calibrationFinished(false, error);
        calibrated = false;
        return;
    }
330

Thiago Santini's avatar
Thiago Santini committed
331 332 333 334
    autoVisualizationTimer.restart();
    emit calibrationFinished(true, "");
}

335
GazeEstimate GazeEstimation::getGazeEstimate(const DataTuple& dataTuple)
Thiago Santini's avatar
Thiago Santini committed
336
{
337 338
    GazeEstimate gazeEstimate;
    if (calibrated && gazeEstimationMethod) {
339
        GazeEstimate left, right, binocular;
340
        gazeEstimationMethod->estimate(dataTuple, left, right, binocular);
341

342
        switch (cfg.inputType) {
343
        case GazeEstimationMethod::BINOCULAR:
344
            gazeEstimate = binocular;
345 346
            break;
        case GazeEstimationMethod::MONO_LEFT:
347
            gazeEstimate = left;
348 349
            break;
        case GazeEstimationMethod::MONO_RIGHT:
350
            gazeEstimate = right;
351
            break;
352
        }
Thiago Santini's avatar
Thiago Santini committed
353
    }
354 355
    return gazeEstimate;
}
Thiago Santini's avatar
Thiago Santini committed
356

357 358 359
void GazeEstimation::estimate(DataTuple dataTuple)
{
    dataTuple.field.gazeEstimate = getGazeEstimate(dataTuple);
Thiago Santini's avatar
Thiago Santini committed
360 361 362 363
    drawGazeEstimationInfo(dataTuple);
    emit gazeEstimationDone(dataTuple);
}

364
void GazeEstimation::printAccuracyInfo(const cv::Mat& errors, const QString& which, const double& diagonal, float& mu, float& sigma)
365 366 367
{
    Scalar m, std;
    meanStdDev(errors, m, std);
368
    qInfo() << which.toLatin1().data() << "Error ( N =" << errors.rows << "):";
369 370
    //qInfo() << QString("m = %1 ( std = %2 ) pixels."
    //    ).arg(m[0], 6, 'f', 2
371 372 373 374
    //    ).arg(std[0], 6, 'f', 2).toLatin1().data();
    mu = m[0] / diagonal;
    sigma = std[0] / diagonal;
    qInfo() << QString("m = %1 ( std = %2 ) % of the image diagonal.").arg(100 * mu, 6, 'f', 2).arg(100 * sigma, 6, 'f', 2).toLatin1().data();
375 376
}

Thiago Santini's avatar
Thiago Santini committed
377 378 379 380 381 382
void GazeEstimation::evaluate()
{
    if (!calibrated)
        return;

    evaluationTuples.clear();
383
    for (unsigned int i = 0; i < collectedTuples.size(); i++)
384
        if (collectedTuples[i].isEvaluation() || (cfg.autoEvaluation && collectedTuples[i].isAutoEval())) {
385
            if (getGazeEstimate(collectedTuples[i]).valid)
386 387
                evaluationTuples.push_back(&collectedTuples[i]);
        }
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424

    centralHullCoverage = 0;
    peripheralHullCoverage = 0;
    if (!interpolationHull.empty()) {
        float delta = 0.25;
        Size size(calibrationTuples[0]->field.width, calibrationTuples[0]->field.height);
        Mat centralRegion = Mat::zeros(size, CV_8U);
        rectangle(centralRegion, Point(delta * size.width, delta * size.height), Point((1 - delta) * size.width, (1 - delta) * size.height), Scalar(255), -1);
        Mat peripheralRegion = 255 - centralRegion;

        Mat hullRegion = Mat::zeros(size, CV_8U);
        vector<vector<Point>> contours;
        contours.push_back(interpolationHull);
        drawContours(hullRegion, contours, 0, Scalar(255), -1);

        vector<Point> nonZero;
        findNonZero(centralRegion, nonZero);
        float centralPixels = nonZero.size();
        findNonZero(peripheralRegion, nonZero);
        float peripheralPixels = nonZero.size();

        Mat intersection;
        bitwise_and(hullRegion, centralRegion, intersection);
        findNonZero(intersection, nonZero);
        centralHullCoverage = nonZero.size() / centralPixels;

        bitwise_and(hullRegion, peripheralRegion, intersection);
        findNonZero(intersection, nonZero);
        peripheralHullCoverage = nonZero.size() / peripheralPixels;

        qInfo() << QString("Central Interpolation Coverage: %1 %").arg(100 * centralHullCoverage, 0, 'f', 2).toLatin1().data();
        qInfo() << QString("Peripheral Interpolation Coverage: %1 %").arg(100 * peripheralHullCoverage, 0, 'f', 2).toLatin1().data();
    }

    meanEvaluationError = 0;
    stdEvaluationError = 0;
    evaluationRegionCoverage = 1;
Thiago Santini's avatar
Thiago Santini committed
425 426

    if (evaluationTuples.size() <= 0 || calibrationTuples.size() <= 0)
427
        return;
Thiago Santini's avatar
Thiago Santini committed
428

429
    // evaluate for known points
Thiago Santini's avatar
Thiago Santini committed
430
    errorVectors.clear();
431
    Mat errors;
432
    for (unsigned int i = 0; i < evaluationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
433
        Point3f gt = evaluationTuples[i]->field.collectionMarker.center;
434
        Point3f gaze = to3D(getGazeEstimate(*evaluationTuples[i]).gp);
Thiago Santini's avatar
Thiago Santini committed
435
        errorVectors.push_back(ErrorVector(gt, gaze));
436
        errors.push_back(errorVectors.back().magnitude());
Thiago Santini's avatar
Thiago Santini committed
437 438
    }

439 440
    double imDiagonal = sqrt(pow(evaluationTuples[0]->field.width, 2) + pow(evaluationTuples[0]->field.height, 2));
    printAccuracyInfo(errors, "Gaze Evaluation", imDiagonal, meanEvaluationError, stdEvaluationError);
441 442

    if (cfg.autoEvaluation) {
443
        double evaluationRegionsCount = pow(1 + 2 * cfg.granularity, 2);
444
        double evaluationRegionsSelected = 0;
445 446
        for (unsigned int i = 0; i < collectedTuples.size(); i++)
            if (collectedTuples[i].isAutoEval())
447
                evaluationRegionsSelected++;
448 449 450
        evaluationRegionCoverage = evaluationRegionsSelected / evaluationRegionsCount;
        qInfo() << QString("Auto Evaluation Region Coverage: %1 %").arg(100 * evaluationRegionCoverage, 0, 'f', 2).toLatin1().data();
    }
Thiago Santini's avatar
Thiago Santini committed
451 452 453 454 455
}

/*
 * File loading / saving
 */
456 457
template <typename T>
static void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key, T& data)
Thiago Santini's avatar
Thiago Santini committed
458 459 460 461 462 463
{
    if (idxMap.count(key) == 0)
        return;

    data = QVariant(tokens[idxMap[key]]).value<T>();
}
464
template <>
465
inline void addData(map<QString, uint>& idxMap, QStringList& tokens, QString key, vector<Marker>& markers)
466 467 468 469 470 471 472 473 474 475 476
{
    if (idxMap.count(key) == 0)
        return;

    QString markersStr = tokens[idxMap[key]];
    for (auto& markerStr : markersStr.split(Token::MarkerEnd)) {
        Marker tmp;
        if (tmp.parse(markerStr))
            markers.push_back(std::move(tmp));
    }
}
Thiago Santini's avatar
Thiago Santini committed
477 478 479
void GazeEstimation::loadTuplesFromFile(CollectionTuple::TupleType tupleType, QString fileName)
{
    vector<CollectionTuple> tuples;
480 481 482 483 484 485 486
    TSVReader tsv;
    if (!tsv.open(fileName))
        return;
    for (int i = 0; i < tsv.size(); i++) {
        CollectionTuple tuple = CollectionTuple(DataTuple(tsv, i));
        tuple.tupleType = tupleType;
        tuples.emplace_back(tuple);
Thiago Santini's avatar
Thiago Santini committed
487 488 489 490 491 492 493
    }
    addTuples(tuples);
    calibrate();
}

void GazeEstimation::saveTuplesToFile(CollectionTuple::TupleType tupleType, QString fileName)
{
494 495 496 497 498 499 500 501 502 503
    vector<CollectionTuple*> pertinentTuples;
    for (unsigned int i = 0; i < collectedTuples.size(); i++) {
        //if ( tupleType == CollectionTuple::CALIBRATION)
        //	if (collectedTuples[i].isEvaluation())
        //		continue;

        if (tupleType == CollectionTuple::EVALUATION) {
            if (collectedTuples[i].isCalibration())
                continue;
        }
Thiago Santini's avatar
Thiago Santini committed
504

505 506
        pertinentTuples.push_back(&collectedTuples[i]);
    }
Thiago Santini's avatar
Thiago Santini committed
507

508
    saveTuplesToFile(pertinentTuples, fileName, QIODevice::WriteOnly);
Thiago Santini's avatar
Thiago Santini committed
509 510
}

511
void GazeEstimation::saveTuplesToFile(const std::vector<CollectionTuple*>& tuples, QString fileName, QFlags<QIODevice::OpenModeFlag> flags)
Thiago Santini's avatar
Thiago Santini committed
512 513 514 515 516
{
    if (fileName.isEmpty() || tuples.size() == 0)
        return;

    QString buffer;
517 518 519
    buffer.append(tuples[0]->header() + Token::Newline);
    for (unsigned int i = 0; i < tuples.size(); i++)
        buffer.append(tuples[i]->toQString() + Token::Newline);
Thiago Santini's avatar
Thiago Santini committed
520 521 522 523 524 525 526 527 528 529 530 531

    QFile f(fileName);
    f.open(flags);
    QTextStream s(&f);
    if (s.status() == QTextStream::Ok)
        s << buffer.toStdString().c_str();
    f.close();
}

void GazeEstimation::updateConfig()
{
    cfg.load(settings);
532
    for (int i = 0; i < availableGazeEstimationMethods.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
533
        QString name = availableGazeEstimationMethods[i]->description().c_str();
Thiago Santini's avatar
Thiago Santini committed
534
        if (cfg.gazeEstimationMethod == name) {
Thiago Santini's avatar
Thiago Santini committed
535
            gazeEstimationMethod = availableGazeEstimationMethods[i];
Thiago Santini's avatar
Thiago Santini committed
536 537 538 539 540 541 542 543 544
            break;
        }
    }
    if (!gazeEstimationMethod) {
        qWarning() << "Unknown gaze estimation method:" << cfg.gazeEstimationMethod;
        if (availableGazeEstimationMethods.size() > 0) {
            gazeEstimationMethod = availableGazeEstimationMethods.front();
            qWarning() << "Defaulting to" << gazeEstimationMethod->description().c_str();
        }
Thiago Santini's avatar
Thiago Santini committed
545 546 547 548
    }
    calibrate();
}

549
void GazeEstimation::selectEvaluationTuples(const int g, const double dx, const double dy, const double rf)
Thiago Santini's avatar
Thiago Santini committed
550 551 552 553 554 555 556 557 558 559
{
    vector<CollectionTuple*> evaluationCandidates;
    vector<Point> points;
    CollectionTuple* t;

    evaluationRegions.clear();

    /*
     * collect non-outlier candidates
     */
560
    for (unsigned int i = 0; i < calibrationTuples.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
561 562 563 564 565
        t = calibrationTuples[i];
        t->autoEval = CollectionTuple::AE_NO;
        if (t->isOutlier())
            continue;
        evaluationCandidates.push_back(t);
566
        points.push_back(to2D(t->field.collectionMarker.center));
Thiago Santini's avatar
Thiago Santini committed
567 568 569 570
    }

    if (evaluationCandidates.size() <= 0)
        return;
571
    t = evaluationCandidates[0];
Thiago Santini's avatar
Thiago Santini committed
572 573 574 575

    /*
     *  exclude points in the hull so we don't lose interpolation range
     */
Thiago Santini's avatar
Thiago Santini committed
576
    const float minDistFromHullPx = 3;
Thiago Santini's avatar
Thiago Santini committed
577 578 579 580
    convexHull(points, interpolationHull);

//#define DBG_AUTOEVAL
#ifdef DBG_AUTOEVAL
581 582 583 584 585 586 587
    const Point* ep[1] = { &interpolationHull[0] };
    int epn = (int)interpolationHull.size();
    Mat dbg = Mat::zeros(evaluationCandidates[0]->field.height, evaluationCandidates[0]->field.width, CV_8UC3);
    fillPoly(dbg, ep, &epn, 1, Scalar(255, 255, 255));
    for (size_t i = evaluationCandidates.size(); i-- > 0;) {
        Point gaze = to2D(evaluationCandidates[i]->field.collectionMarker.center);
        circle(dbg, gaze, 5, Scalar(0, 255, 255), -1);
Thiago Santini's avatar
Thiago Santini committed
588 589 590
    }
#endif

591 592 593
    for (size_t i = evaluationCandidates.size(); i-- > 0;) {
        Point gaze = to2D(evaluationCandidates[i]->field.collectionMarker.center);
        for (unsigned int j = 0; j < interpolationHull.size(); j++) {
Thiago Santini's avatar
Thiago Santini committed
594
            if (ED(gaze, interpolationHull[j]) <= minDistFromHullPx) {
Thiago Santini's avatar
Thiago Santini committed
595
#ifdef DBG_AUTOEVAL
596
                circle(dbg, gaze, 5, Scalar(255, 0, 0), -1);
Thiago Santini's avatar
Thiago Santini committed
597
#endif
598
                evaluationCandidates.erase(evaluationCandidates.begin() + i);
Thiago Santini's avatar
Thiago Santini committed
599 600 601 602 603 604 605 606
                break;
            }
        }
    }

    /*
     * Select points that minimize distance to evaluation region center for evaluation
     */
607 608 609 610 611
    Point center(t->field.width / 2, t->field.height / 2);
    int dxPx = dx * t->field.width;
    int dyPx = dy * t->field.height;
    int w = rf * dxPx;
    int h = rf * dyPx;
Thiago Santini's avatar
Thiago Santini committed
612 613 614
    vector<Point> evaluationPoints;
    for (int x = -g; x <= g; x++) {
        for (int y = -g; y <= g; y++) {
615 616
            EvaluationRegion er(center.x + x * dxPx, center.y + y * dyPx, w, h);
            for (int i = 0; i < evaluationCandidates.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
                er.eval(evaluationCandidates[i]);
            if (er.selected) {
                er.selected->autoEval = CollectionTuple::AE_YES;
                evaluationPoints.push_back(to2D(er.selected->field.collectionMarker.center));
            }
            evaluationRegions.push_back(er);
#ifdef DBG_AUTOEVAL
            er.draw(dbg);
#endif
        }
    }

    /*
     * Mark calibration points that could bias the auto evaluation
     */
Thiago Santini's avatar
Thiago Santini committed
632
    const float minDistFromEvaluationPx = 5;
633
    for (unsigned int i = 0; i < evaluationPoints.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
634
        Point ep = evaluationPoints[i];
635
        for (unsigned int j = 0; j < evaluationCandidates.size(); j++) {
Thiago Santini's avatar
Thiago Santini committed
636 637 638
            t = evaluationCandidates[j];
            if (t->autoEval == CollectionTuple::AE_NO) {
                Point cp = to2D(t->field.collectionMarker.center);
Thiago Santini's avatar
Thiago Santini committed
639
                if (ED(ep, cp) < minDistFromEvaluationPx) {
Thiago Santini's avatar
Thiago Santini committed
640 641
                    t->autoEval = CollectionTuple::AE_BIASED;
#ifdef DBG_AUTOEVAL
642
                    circle(dbg, cp, 2, Scalar(255, 255, 0), -1);
Thiago Santini's avatar
Thiago Santini committed
643 644 645 646 647 648 649 650 651 652 653
#endif
                }
            }
        }
    }

#ifdef DBG_AUTOEVAL
    imshow("dbg", dbg);
#endif
}

654
void GazeEstimation::drawGazeEstimationInfo(DataTuple& dataTuple)
Thiago Santini's avatar
Thiago Santini committed
655 656 657 658 659 660 661 662 663 664 665 666 667
{
    dataTuple.showGazeEstimationVisualization = false;

    // TODO: improve this? maybe draw on an overlay canvas that is only updated
    // when something changes. Then we overlay the canvas on a copy of the image
    if (dataTuple.field.input.empty() || !cfg.visualize)
        return;

    bool shouldDisplay = false;
    if (isCalibrating)
        shouldDisplay = true;
    else {
        if (autoVisualizationTimer.isValid()) {
668
            if (autoVisualizationTimer.elapsed() < cfg.visualizationTimeS * 1e3)
Thiago Santini's avatar
Thiago Santini committed
669 670 671 672 673
                shouldDisplay = true;
            else
                autoVisualizationTimer.invalidate();
        }
    }
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
    if (!shouldDisplay) {
        lastOverlayIdx = 0;
        return;
    }

    dataTuple.showGazeEstimationVisualization = true;
    dataTuple.gazeEstimationVisualization = vis;

    // avoid drawing every single frame
    //static Timestamp lastGazeEstimationVisualizationTimestamp = 0;
    //Timestamp current = gTimer.elapsed();
    //bool shouldDraw = current - lastGazeEstimationVisualizationTimestamp > 40;
    //if (!shouldDraw)
    //	return;
    //lastGazeEstimationVisualizationTimestamp = current;

    vis = dataTuple.field.input.clone();
    int r = max<int>(1, 0.003125 * max<int>(vis.rows, vis.cols));

    if (isCalibrating) {

        if (lastOverlayIdx > collectedTuples.size()) // sample removed, restart
            lastOverlayIdx = 0;

        if (lastOverlayIdx == 0)
            overlay = Mat::zeros(vis.rows, vis.cols, CV_8UC3);

        for (; lastOverlayIdx < collectedTuples.size(); lastOverlayIdx++) {
            circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r, CV_ALMOST_BLACK, -1);
            if (collectedTuples[lastOverlayIdx].isCalibration())
                circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r + 1, CV_GREEN, 0.5 * r);
            else
                circle(overlay, to2D(collectedTuples[lastOverlayIdx].field.collectionMarker.center), r + 1, CV_CYAN, 0.5 * r);
        }

    } else {
        // Calibration finished; overlay results and restart
        overlay = Mat::zeros(vis.rows, vis.cols, CV_8UC3);
        lastOverlayIdx = 0;

        if (!interpolationHull.empty()) {
            vector<vector<Point>> contours;
Thiago Santini's avatar
Thiago Santini committed
716
            contours.push_back(interpolationHull);
717
            drawContours(overlay, contours, 0, CV_GREEN, r);
Thiago Santini's avatar
Thiago Santini committed
718 719
        }

720 721
        for (size_t i = 0; i < errorVectors.size(); i++)
            errorVectors[i].draw(overlay, 2, CV_RED);
Thiago Santini's avatar
Thiago Santini committed
722 723

        if (cfg.autoEvaluation)
724 725
            for (size_t i = 0; i < evaluationRegions.size(); i++)
                evaluationRegions[i].draw(overlay, r);
Thiago Santini's avatar
Thiago Santini committed
726

727 728 729 730 731 732 733
        for (size_t i = 0; i < collectedTuples.size(); i++) {
            if (!collectedTuples[i].isCalibration()) {
                circle(overlay, to2D(collectedTuples[i].field.collectionMarker.center), r, CV_BLACK, -1);
                circle(overlay, to2D(collectedTuples[i].field.collectionMarker.center), r + 1, CV_CYAN, 0.5 * r);
            }
        }
    }
Thiago Santini's avatar
Thiago Santini committed
734

735 736 737
    // Overlay on visualization image; notice we use CV_ALMOST_BLACK instead of
    // CV_BLACK so we don't need to create an additional mask :-)
    overlay.copyTo(vis, overlay);
Thiago Santini's avatar
Thiago Santini committed
738 739 740 741 742 743 744
}

void GazeEstimation::setCalibrating(bool v)
{
    isCalibrating = v;
}

745
void GazeEstimation::updateInterpolationHull(const std::vector<CollectionTuple>& tuples)
746 747 748 749 750
{
    if (tuples.size() <= 0)
        return;

    vector<Point> points;
751 752
    for (unsigned int i = 0; i < tuples.size(); i++)
        points.push_back(to2D(tuples[i].field.collectionMarker.center));
753 754 755

    convexHull(points, interpolationHull);
}
756 757 758

void GazeEstimation::updateTemporalGazes(std::vector<CollectionTuple>& tuples)
{
Thiago Santini's avatar
Thiago Santini committed
759
    (void)tuples;
760
}