GazeEstimation.h 9.21 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3 4 5
#ifndef GAZEESTIMATION_H
#define GAZEESTIMATION_H

#include <deque>
#include <map>
6
#include <memory>
7
#include <vector>
Thiago Santini's avatar
Thiago Santini committed
8 9 10

#include <QObject>

11 12
#include "Reference.h"
#include "data/DataTuple.h"
Thiago Santini's avatar
Thiago Santini committed
13
#include "gaze-estimation/BinocularPolyFit.h"
Thiago Santini's avatar
Thiago Santini committed
14
#include "gaze-estimation/GazeEstimationMethod.h"
15
#include "gaze-estimation/GazeVectorBinocularPolyFit.h"
Thiago Santini's avatar
Thiago Santini committed
16
#include "gaze-estimation/Homography.h"
17
#include "gaze-estimation/PolyFit.h"
18
#include "globals.h"
19
#include "post-processing/TSVReader.h"
Thiago Santini's avatar
Thiago Santini committed
20 21 22 23 24 25 26

/*******************************************************************************
 * Helper Classes
 ******************************************************************************/
class EvaluationRegion {
public:
    EvaluationRegion(int x, int y, int w, int h)
27 28 29 30 31 32 33 34 35 36
        : x(x)
        , y(y)
        , w(w)
        , h(h)
        , rw2(pow(w / 2.0, 2))
        , rh2(pow(h / 2.0, 2))
        , mn(DBL_MAX)
        , selected(nullptr)
    {
    }
Thiago Santini's avatar
Thiago Santini committed
37 38
    int x, y, w, h, rw2, rh2;
    double mn;
39 40 41 42 43 44 45 46 47
    CollectionTuple* selected;
    void eval(CollectionTuple* t)
    {
        cv::Point p = to2D(t->field.collectionMarker.center);
        double d = (pow(p.x - x, 2) / rw2 + pow(p.y - y, 2) / rh2);
        if (d <= 1 && d < mn) {
            mn = d;
            selected = t;
        }
Thiago Santini's avatar
Thiago Santini committed
48
    }
49 50 51 52 53 54 55
    void draw(cv::Mat& in, int r = 3)
    {
        cv::RotatedRect el(cv::Point(x, y), cv::Size(w, h), 0);
        if (selected) {
            cv::ellipse(in, el, CV_GREEN, 0.5 * r);
            cv::circle(in, to2D(selected->field.collectionMarker.center), r, CV_ALMOST_BLACK, -1);
            cv::circle(in, to2D(selected->field.collectionMarker.center), r + 1, CV_GREEN, r);
Thiago Santini's avatar
Thiago Santini committed
56
        } else
57
            cv::ellipse(in, el, CV_RED, 0.5 * r);
Thiago Santini's avatar
Thiago Santini committed
58 59 60 61 62
    }
};

class ErrorVector {
public:
63 64 65 66 67
    ErrorVector(cv::Point3f gt, cv::Point3f gaze)
        : gt(gt)
        , gaze(gaze)
    {
    }
Thiago Santini's avatar
Thiago Santini committed
68 69 70

    cv::Point3f gt;
    cv::Point3f gaze;
71 72 73
    void draw(cv::Mat& in, int w = 2, cv::Scalar color = CV_RED)
    {
        cv::line(in, cv::Point(gt.x, gt.y), cv::Point(gaze.x, gaze.y), color, w);
Thiago Santini's avatar
Thiago Santini committed
74
    }
75
    double magnitude() { return cv::norm(gt - gaze); }
Thiago Santini's avatar
Thiago Santini committed
76 77 78 79 80
};

/*******************************************************************************
 * Gaze Estimation
 ******************************************************************************/
81
class GazeEstimationConfig {
Thiago Santini's avatar
Thiago Santini committed
82
public:
83 84 85 86 87 88 89
    GazeEstimationConfig()
        : samplingTimeMs(500)
        , useMedian(false)
        , removeOutliers(true)
        , pupilRatioOutliers(true)
        , pupilPositionOutliers(true)
        , pupilOutlineOutliers(true)
90
        , pupilDiameterOutliers(true)
91
        , inputType(GazeEstimationMethod::BINOCULAR)
92
        , gazeEstimationMethod("BIPOLY_X_Y_XY_XX_YY_XYY_YXX_XXYY")
93 94
        , visualize(true)
        , visualizationTimeS(5)
Thiago Santini's avatar
Thiago Santini committed
95
        , showVirtualMarker(false)
96 97 98 99 100 101 102
        , autoEvaluation(true)
        , granularity(2)
        , horizontalStride(0.15)
        , verticalStride(0.15)
        , rangeFactor(0.95)
        , minCentralAreaCoverage(0.0f)
        , minPeriphericAreaCoverage(0.0f)
103
        , maxReprojectionError(0.0f)
104 105
    {
    }
Thiago Santini's avatar
Thiago Santini committed
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120

    /*
     * Gaze Estimation Widget
     */
    int samplingTimeMs;
    bool useMedian;

    /*
     * Gaze Estimation
     */
    // outlier removal
    bool removeOutliers;
    bool pupilRatioOutliers;
    bool pupilPositionOutliers;
    bool pupilOutlineOutliers;
121
    bool pupilDiameterOutliers;
Thiago Santini's avatar
Thiago Santini committed
122 123 124 125 126 127 128 129 130 131

    // Method
    GazeEstimationMethod::InputType inputType;
    QString gazeEstimationMethod;

    // Visualization
    bool visualize;
    int visualizationTimeS;

    // CalibMe
Thiago Santini's avatar
Thiago Santini committed
132
    bool showVirtualMarker;
Thiago Santini's avatar
Thiago Santini committed
133 134 135 136 137 138
    bool autoEvaluation;
    int granularity;
    double horizontalStride;
    double verticalStride;
    double rangeFactor;

139 140 141 142
    // Quality check
    float minCentralAreaCoverage;
    float minPeriphericAreaCoverage;
    float maxReprojectionError;
Thiago Santini's avatar
Thiago Santini committed
143

144
    void save(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
145 146 147 148 149 150 151 152 153 154 155 156 157 158
    {
        settings->sync();
        settings->setValue("samplingTimeMs", samplingTimeMs);
        settings->setValue("useMedian", useMedian);
        settings->setValue("inputType", inputType);
        settings->setValue("gazeEstimationMethod", gazeEstimationMethod);
        settings->setValue("visualize", visualize);
        settings->setValue("visualizationTimeS", visualizationTimeS);

        // CalibMe
        settings->setValue("CalibMe/removeOutliers", removeOutliers);
        settings->setValue("CalibMe/pupilRatioOutliers", pupilRatioOutliers);
        settings->setValue("CalibMe/pupilPositionOutliers", pupilPositionOutliers);
        settings->setValue("CalibMe/pupilOutlineOutliers", pupilOutlineOutliers);
159
        settings->setValue("CalibMe/pupilDiameterOutliers", pupilDiameterOutliers);
Thiago Santini's avatar
Thiago Santini committed
160
        settings->setValue("CalibMe/showVirtualMarker", showVirtualMarker);
Thiago Santini's avatar
Thiago Santini committed
161 162 163 164
        settings->setValue("CalibMe/autoEvaluation", autoEvaluation);
        settings->setValue("CalibMe/granularity", granularity);
        settings->setValue("CalibMe/horizontalStride", horizontalStride);
        settings->setValue("CalibMe/verticalStride", verticalStride);
165
        settings->setValue("CalibMe/rangeFactor", rangeFactor);
166

167 168 169 170 171
        // Quality check
        settings->setValue("quality/minCentralAreaCoverage", minCentralAreaCoverage);
        settings->setValue("quality/minPeriphericAreaCoverage", minPeriphericAreaCoverage);
        settings->setValue("quality/maxReprojectionError", maxReprojectionError);
    }
Thiago Santini's avatar
Thiago Santini committed
172

173
    void load(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
174 175 176 177 178
    {
        settings->sync();
        set(settings, "samplingTimeMs", samplingTimeMs);
        set(settings, "useMedian", useMedian);
        set(settings, "inputType", inputType);
179
        set(settings, "gazeEstimationMethod", gazeEstimationMethod);
Thiago Santini's avatar
Thiago Santini committed
180 181 182 183
        set(settings, "visualize", visualize);
        set(settings, "visualizationTimeS", visualizationTimeS);

        //CalibMe
184 185
        set(settings, "CalibMe/removeOutliers", removeOutliers);
        set(settings, "CalibMe/pupilRatioOutliers", pupilRatioOutliers);
Thiago Santini's avatar
Thiago Santini committed
186
        set(settings, "CalibMe/pupilPositionOutliers", pupilPositionOutliers);
187
        set(settings, "CalibMe/pupilOutlineOutliers", pupilOutlineOutliers);
188
        set(settings, "CalibMe/pupilDiameterOutliers", pupilDiameterOutliers);
Thiago Santini's avatar
Thiago Santini committed
189
        set(settings, "CalibMe/showVirtualMarker", showVirtualMarker);
190 191 192 193 194 195 196 197 198 199
        set(settings, "CalibMe/autoEvaluation", autoEvaluation);
        set(settings, "CalibMe/granularity", granularity);
        set(settings, "CalibMe/horizontalStride", horizontalStride);
        set(settings, "CalibMe/verticalStride", verticalStride);
        set(settings, "CalibMe/rangeFactor", rangeFactor);

        set(settings, "quality/minCentralAreaCoverage", minCentralAreaCoverage);
        set(settings, "quality/minPeriphericAreaCoverage", minPeriphericAreaCoverage);
        set(settings, "quality/maxReprojectionError", maxReprojectionError);
    }
Thiago Santini's avatar
Thiago Santini committed
200 201
};

202
class GazeEstimation : public QObject {
Thiago Santini's avatar
Thiago Santini committed
203 204 205
    Q_OBJECT

public:
206 207 208
    explicit GazeEstimation(QObject* parent = 0);
    virtual ~GazeEstimation();
    QSettings* settings;
209
    std::vector<std::shared_ptr<GazeEstimationMethod>> availableGazeEstimationMethods;
Thiago Santini's avatar
Thiago Santini committed
210 211 212 213 214 215 216 217

signals:
    void gazeEstimationDone(DataTuple dataTuple);
    void calibrationFinished(bool status, QString msg);

public slots:
    void loadTuplesFromFile(CollectionTuple::TupleType type, QString fileName);
    void saveTuplesToFile(CollectionTuple::TupleType type, QString fileName);
218
    void saveTuplesToFile(const std::vector<CollectionTuple*>& tuples, QString fileName, QFlags<QIODevice::OpenModeFlag> flags);
Thiago Santini's avatar
Thiago Santini committed
219 220
    void addTuple(CollectionTuple tuple);
    void addTuples(std::vector<CollectionTuple> tuples);
221 222
    void reset(CollectionTuple::TupleType type);
    void saveCalibration() { saveTuplesToFile(CollectionTuple::TupleType::CALIBRATION, QString("%1-calibration.tup").arg(gTimer.elapsed())); }
Thiago Santini's avatar
Thiago Santini committed
223 224 225

    void setCalibrating(bool v);

226
    void updateTemporalGazes(std::vector<CollectionTuple>& tuples);
Thiago Santini's avatar
Thiago Santini committed
227
    void calibrate();
228
    void estimate(DataTuple dataTuple);
Thiago Santini's avatar
Thiago Santini committed
229
    void evaluate();
230
    void printAccuracyInfo(const cv::Mat& errors, const QString& which, const double& diagonal, float& mu, float& sigma);
Thiago Santini's avatar
Thiago Santini committed
231 232 233 234 235 236 237 238 239 240 241

    void updateConfig();

private:
    bool calibrated;
    bool isCalibrating;
    GazeEstimationConfig cfg;
    std::vector<CollectionTuple> collectedTuples;
    std::vector<CollectionTuple*> calibrationTuples;
    std::vector<CollectionTuple*> evaluationTuples;

242
    std::shared_ptr<GazeEstimationMethod> gazeEstimationMethod;
Thiago Santini's avatar
Thiago Santini committed
243 244
    QMutex cfgMutex;

245 246 247 248 249
    float centralHullCoverage;
    float peripheralHullCoverage;
    float evaluationRegionCoverage;
    float meanEvaluationError;
    float stdEvaluationError;
250

251
    std::vector<cv::Point> interpolationHull;
Thiago Santini's avatar
Thiago Santini committed
252 253 254 255
    std::vector<EvaluationRegion> evaluationRegions;
    std::vector<ErrorVector> errorVectors;
    QElapsedTimer autoVisualizationTimer;
    cv::Mat vis;
256 257
    cv::Mat overlay;
    int lastOverlayIdx;
258
    GazeEstimate getGazeEstimate(const DataTuple& dataTuple);
Thiago Santini's avatar
Thiago Santini committed
259 260 261

private slots:
    void detectOutliers();
262 263 264
    bool isPupilRatioValid(const EyeData& prev, const EyeData& cur);
    void findPupilPositionOutliers(const cv::Mat& pupil, cv::Mat& mask);
    bool isPupilOutlineValid(const EyeData& cur);
Thiago Santini's avatar
Thiago Santini committed
265 266
    void selectEvaluationTuples(const int g, const double dx, const double dy, const double rf);

267 268
    void updateInterpolationHull(const std::vector<CollectionTuple>& tuples);
    void drawGazeEstimationInfo(DataTuple& dataTuple);
Thiago Santini's avatar
Thiago Santini committed
269 270 271
};

#endif // GAZEESTIMATION_H