GazeEstimation.h 8.99 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3 4 5
#ifndef GAZEESTIMATION_H
#define GAZEESTIMATION_H

#include <deque>
#include <map>
6
#include <memory>
7
#include <vector>
Thiago Santini's avatar
Thiago Santini committed
8 9 10

#include <QObject>

11 12
#include "Reference.h"
#include "data/DataTuple.h"
Thiago Santini's avatar
Thiago Santini committed
13
#include "gaze-estimation/BinocularPolyFit.h"
Thiago Santini's avatar
Thiago Santini committed
14 15
#include "gaze-estimation/GazeEstimationMethod.h"
#include "gaze-estimation/Homography.h"
16
#include "gaze-estimation/PolyFit.h"
17
#include "globals.h"
Thiago Santini's avatar
Thiago Santini committed
18 19 20 21 22 23 24

/*******************************************************************************
 * Helper Classes
 ******************************************************************************/
class EvaluationRegion {
public:
    EvaluationRegion(int x, int y, int w, int h)
25 26 27 28 29 30 31 32 33 34
        : x(x)
        , y(y)
        , w(w)
        , h(h)
        , rw2(pow(w / 2.0, 2))
        , rh2(pow(h / 2.0, 2))
        , mn(DBL_MAX)
        , selected(nullptr)
    {
    }
Thiago Santini's avatar
Thiago Santini committed
35 36
    int x, y, w, h, rw2, rh2;
    double mn;
37 38 39 40 41 42 43 44 45
    CollectionTuple* selected;
    void eval(CollectionTuple* t)
    {
        cv::Point p = to2D(t->field.collectionMarker.center);
        double d = (pow(p.x - x, 2) / rw2 + pow(p.y - y, 2) / rh2);
        if (d <= 1 && d < mn) {
            mn = d;
            selected = t;
        }
Thiago Santini's avatar
Thiago Santini committed
46
    }
47 48 49 50 51 52 53
    void draw(cv::Mat& in, int r = 3)
    {
        cv::RotatedRect el(cv::Point(x, y), cv::Size(w, h), 0);
        if (selected) {
            cv::ellipse(in, el, CV_GREEN, 0.5 * r);
            cv::circle(in, to2D(selected->field.collectionMarker.center), r, CV_ALMOST_BLACK, -1);
            cv::circle(in, to2D(selected->field.collectionMarker.center), r + 1, CV_GREEN, r);
Thiago Santini's avatar
Thiago Santini committed
54
        } else
55
            cv::ellipse(in, el, CV_RED, 0.5 * r);
Thiago Santini's avatar
Thiago Santini committed
56 57 58 59 60
    }
};

class ErrorVector {
public:
61 62 63 64 65
    ErrorVector(cv::Point3f gt, cv::Point3f gaze)
        : gt(gt)
        , gaze(gaze)
    {
    }
Thiago Santini's avatar
Thiago Santini committed
66 67 68

    cv::Point3f gt;
    cv::Point3f gaze;
69 70 71
    void draw(cv::Mat& in, int w = 2, cv::Scalar color = CV_RED)
    {
        cv::line(in, cv::Point(gt.x, gt.y), cv::Point(gaze.x, gaze.y), color, w);
Thiago Santini's avatar
Thiago Santini committed
72
    }
73
    double magnitude() { return cv::norm(gt - gaze); }
Thiago Santini's avatar
Thiago Santini committed
74 75 76 77 78
};

/*******************************************************************************
 * Gaze Estimation
 ******************************************************************************/
79
class GazeEstimationConfig {
Thiago Santini's avatar
Thiago Santini committed
80
public:
81 82 83 84 85 86 87
    GazeEstimationConfig()
        : samplingTimeMs(500)
        , useMedian(false)
        , removeOutliers(true)
        , pupilRatioOutliers(true)
        , pupilPositionOutliers(true)
        , pupilOutlineOutliers(true)
88
        , pupilDiameterOutliers(true)
89
        , inputType(GazeEstimationMethod::BINOCULAR)
90
        , gazeEstimationMethod("BIPOLY_X_Y_XY_XX_YY_XYY_YXX_XXYY")
91 92
        , visualize(true)
        , visualizationTimeS(5)
Thiago Santini's avatar
Thiago Santini committed
93
        , showVirtualMarker(false)
94 95 96 97 98 99 100 101 102 103
        , autoEvaluation(true)
        , granularity(2)
        , horizontalStride(0.15)
        , verticalStride(0.15)
        , rangeFactor(0.95)
        , minCentralAreaCoverage(0.0f)
        , minPeriphericAreaCoverage(0.0f)
        , maxReprojectionError(10.0f)
    {
    }
Thiago Santini's avatar
Thiago Santini committed
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118

    /*
     * Gaze Estimation Widget
     */
    int samplingTimeMs;
    bool useMedian;

    /*
     * Gaze Estimation
     */
    // outlier removal
    bool removeOutliers;
    bool pupilRatioOutliers;
    bool pupilPositionOutliers;
    bool pupilOutlineOutliers;
119
    bool pupilDiameterOutliers;
Thiago Santini's avatar
Thiago Santini committed
120 121 122 123 124 125 126 127 128 129

    // Method
    GazeEstimationMethod::InputType inputType;
    QString gazeEstimationMethod;

    // Visualization
    bool visualize;
    int visualizationTimeS;

    // CalibMe
Thiago Santini's avatar
Thiago Santini committed
130
    bool showVirtualMarker;
Thiago Santini's avatar
Thiago Santini committed
131 132 133 134 135 136
    bool autoEvaluation;
    int granularity;
    double horizontalStride;
    double verticalStride;
    double rangeFactor;

137 138 139 140
    // Quality check
    float minCentralAreaCoverage;
    float minPeriphericAreaCoverage;
    float maxReprojectionError;
Thiago Santini's avatar
Thiago Santini committed
141

142
    void save(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
143 144 145 146 147 148 149 150 151 152 153 154 155 156
    {
        settings->sync();
        settings->setValue("samplingTimeMs", samplingTimeMs);
        settings->setValue("useMedian", useMedian);
        settings->setValue("inputType", inputType);
        settings->setValue("gazeEstimationMethod", gazeEstimationMethod);
        settings->setValue("visualize", visualize);
        settings->setValue("visualizationTimeS", visualizationTimeS);

        // CalibMe
        settings->setValue("CalibMe/removeOutliers", removeOutliers);
        settings->setValue("CalibMe/pupilRatioOutliers", pupilRatioOutliers);
        settings->setValue("CalibMe/pupilPositionOutliers", pupilPositionOutliers);
        settings->setValue("CalibMe/pupilOutlineOutliers", pupilOutlineOutliers);
157
        settings->setValue("CalibMe/pupilDiameterOutliers", pupilDiameterOutliers);
Thiago Santini's avatar
Thiago Santini committed
158
        settings->setValue("CalibMe/showVirtualMarker", showVirtualMarker);
Thiago Santini's avatar
Thiago Santini committed
159 160 161 162
        settings->setValue("CalibMe/autoEvaluation", autoEvaluation);
        settings->setValue("CalibMe/granularity", granularity);
        settings->setValue("CalibMe/horizontalStride", horizontalStride);
        settings->setValue("CalibMe/verticalStride", verticalStride);
163
        settings->setValue("CalibMe/rangeFactor", rangeFactor);
164

165 166 167 168 169
        // Quality check
        settings->setValue("quality/minCentralAreaCoverage", minCentralAreaCoverage);
        settings->setValue("quality/minPeriphericAreaCoverage", minPeriphericAreaCoverage);
        settings->setValue("quality/maxReprojectionError", maxReprojectionError);
    }
Thiago Santini's avatar
Thiago Santini committed
170

171
    void load(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
172 173 174 175 176
    {
        settings->sync();
        set(settings, "samplingTimeMs", samplingTimeMs);
        set(settings, "useMedian", useMedian);
        set(settings, "inputType", inputType);
177
        set(settings, "gazeEstimationMethod", gazeEstimationMethod);
Thiago Santini's avatar
Thiago Santini committed
178 179 180 181
        set(settings, "visualize", visualize);
        set(settings, "visualizationTimeS", visualizationTimeS);

        //CalibMe
182 183
        set(settings, "CalibMe/removeOutliers", removeOutliers);
        set(settings, "CalibMe/pupilRatioOutliers", pupilRatioOutliers);
Thiago Santini's avatar
Thiago Santini committed
184
        set(settings, "CalibMe/pupilPositionOutliers", pupilPositionOutliers);
185
        set(settings, "CalibMe/pupilOutlineOutliers", pupilOutlineOutliers);
186
        set(settings, "CalibMe/pupilDiameterOutliers", pupilDiameterOutliers);
Thiago Santini's avatar
Thiago Santini committed
187
        set(settings, "CalibMe/showVirtualMarker", showVirtualMarker);
188 189 190 191 192 193 194 195 196 197
        set(settings, "CalibMe/autoEvaluation", autoEvaluation);
        set(settings, "CalibMe/granularity", granularity);
        set(settings, "CalibMe/horizontalStride", horizontalStride);
        set(settings, "CalibMe/verticalStride", verticalStride);
        set(settings, "CalibMe/rangeFactor", rangeFactor);

        set(settings, "quality/minCentralAreaCoverage", minCentralAreaCoverage);
        set(settings, "quality/minPeriphericAreaCoverage", minPeriphericAreaCoverage);
        set(settings, "quality/maxReprojectionError", maxReprojectionError);
    }
Thiago Santini's avatar
Thiago Santini committed
198 199
};

200
class GazeEstimation : public QObject {
Thiago Santini's avatar
Thiago Santini committed
201 202 203
    Q_OBJECT

public:
204 205 206
    explicit GazeEstimation(QObject* parent = 0);
    virtual ~GazeEstimation();
    QSettings* settings;
207
    std::vector<std::shared_ptr<GazeEstimationMethod>> availableGazeEstimationMethods;
Thiago Santini's avatar
Thiago Santini committed
208 209 210 211 212 213 214 215

signals:
    void gazeEstimationDone(DataTuple dataTuple);
    void calibrationFinished(bool status, QString msg);

public slots:
    void loadTuplesFromFile(CollectionTuple::TupleType type, QString fileName);
    void saveTuplesToFile(CollectionTuple::TupleType type, QString fileName);
216
    void saveTuplesToFile(const std::vector<CollectionTuple*>& tuples, QString fileName, QFlags<QIODevice::OpenModeFlag> flags);
Thiago Santini's avatar
Thiago Santini committed
217 218
    void addTuple(CollectionTuple tuple);
    void addTuples(std::vector<CollectionTuple> tuples);
219 220
    void reset(CollectionTuple::TupleType type);
    void saveCalibration() { saveTuplesToFile(CollectionTuple::TupleType::CALIBRATION, QString("%1-calibration.tup").arg(gTimer.elapsed())); }
Thiago Santini's avatar
Thiago Santini committed
221 222 223 224 225 226

    void setCalibrating(bool v);

    void calibrate();
    void estimate(DataTuple dataTuple);
    void evaluate();
227
    void printAccuracyInfo(const cv::Mat& errors, const QString& which, const double& diagonal, float& mu, float& sigma);
Thiago Santini's avatar
Thiago Santini committed
228 229 230 231 232 233 234 235 236 237 238

    void updateConfig();

private:
    bool calibrated;
    bool isCalibrating;
    GazeEstimationConfig cfg;
    std::vector<CollectionTuple> collectedTuples;
    std::vector<CollectionTuple*> calibrationTuples;
    std::vector<CollectionTuple*> evaluationTuples;

239
    std::shared_ptr<GazeEstimationMethod> gazeEstimationMethod;
Thiago Santini's avatar
Thiago Santini committed
240 241
    QMutex cfgMutex;

242 243 244 245 246
    float centralHullCoverage;
    float peripheralHullCoverage;
    float evaluationRegionCoverage;
    float meanEvaluationError;
    float stdEvaluationError;
247

248
    std::vector<cv::Point> interpolationHull;
Thiago Santini's avatar
Thiago Santini committed
249 250 251 252
    std::vector<EvaluationRegion> evaluationRegions;
    std::vector<ErrorVector> errorVectors;
    QElapsedTimer autoVisualizationTimer;
    cv::Mat vis;
253 254
    cv::Mat overlay;
    int lastOverlayIdx;
Thiago Santini's avatar
Thiago Santini committed
255 256 257

private slots:
    void detectOutliers();
258 259 260
    bool isPupilRatioValid(const EyeData& prev, const EyeData& cur);
    void findPupilPositionOutliers(const cv::Mat& pupil, cv::Mat& mask);
    bool isPupilOutlineValid(const EyeData& cur);
Thiago Santini's avatar
Thiago Santini committed
261 262
    void selectEvaluationTuples(const int g, const double dx, const double dy, const double rf);

263 264
    void updateInterpolationHull(const std::vector<CollectionTuple>& tuples);
    void drawGazeEstimationInfo(DataTuple& dataTuple);
Thiago Santini's avatar
Thiago Santini committed
265 266 267
};

#endif // GAZEESTIMATION_H