EyeImageProcessor.h 8.98 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3
#ifndef EYEIMAGEPROCESSOR_H
#define EYEIMAGEPROCESSOR_H

4 5
#include <memory>

Thiago Santini's avatar
Thiago Santini committed
6
#include <QApplication>
7 8
#include <QCheckBox>
#include <QComboBox>
Thiago Santini's avatar
Thiago Santini committed
9
#include <QDialog>
10
#include <QFormLayout>
Thiago Santini's avatar
Thiago Santini committed
11
#include <QGridLayout>
12
#include <QGroupBox>
Thiago Santini's avatar
Thiago Santini committed
13
#include <QLabel>
14
#include <QObject>
Thiago Santini's avatar
Thiago Santini committed
15 16
#include <QPushButton>
#include <QSettings>
17
#include <QSpinBox>
Thiago Santini's avatar
Thiago Santini committed
18 19 20 21

#include <opencv/cv.h>

#include "InputWidget.h"
22
#include "data/EyeData.h"
Thiago Santini's avatar
Thiago Santini committed
23 24
#include "pupil-detection/ElSe.h"
#include "pupil-detection/ExCuSe.h"
25
#include "pupil-detection/PuRe.h"
Thiago Santini's avatar
Thiago Santini committed
26 27 28 29 30 31
#ifdef STARBURST
#include "pupil-detection/Starburst.h"
#endif
#ifdef SWIRSKI
#include "pupil-detection/Swirski.h"
#endif
32
#include "CameraCalibration.h"
33
#include "globals.h"
34
#include "ocv_utils.h"
35 36 37
#include "pupil-detection/PupilDetectionMethod.h"
#include "pupil-tracking/PuReST.h"
#include "pupil-tracking/PupilTrackingMethod.h"
38
#include "utils.h"
Thiago Santini's avatar
Thiago Santini committed
39

40
class EyeImageProcessorConfig {
Thiago Santini's avatar
Thiago Santini committed
41 42
public:
    EyeImageProcessorConfig()
43 44 45 46 47 48 49
        : inputSize(cv::Size(0, 0))
        , flip(CV_FLIP_NONE)
        , undistort(false)
        , coarseDetection(false)
        , processingDownscalingFactor(1)
        , pupilDetectionMethod(PuRe::desc.c_str())
        , tracking(true)
50 51
        , roi(0.0f, 0.0f, 1.f, 1.f)
        , minPupilDiameterRatio(0.0f)
52
        , maxPupilDiameterRatio(0.27f)
53 54
    {
    }
Thiago Santini's avatar
Thiago Santini committed
55 56 57

    cv::Size inputSize;
    CVFlip flip;
58 59
    bool undistort;
    bool coarseDetection;
Thiago Santini's avatar
Thiago Santini committed
60
    double processingDownscalingFactor;
61 62
    QString pupilDetectionMethod;
    bool tracking;
63 64
    // these are per session!
    cv::Rect2f roi;
65 66
    float minPupilDiameterRatio;
    float maxPupilDiameterRatio;
Thiago Santini's avatar
Thiago Santini committed
67

68
    void save(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
69 70 71 72 73 74
    {
        settings->sync();
        settings->setValue("width", inputSize.width);
        settings->setValue("height", inputSize.height);
        settings->setValue("flip", flip);
        settings->setValue("undistort", undistort);
75 76
        settings->setValue("coarseDetection", coarseDetection);
        settings->setValue("processingDownscalingFactor", processingDownscalingFactor);
Thiago Santini's avatar
Thiago Santini committed
77
        settings->setValue("pupilDetectionMethod", pupilDetectionMethod);
78 79
        settings->setValue("tracking", tracking);
    }
Thiago Santini's avatar
Thiago Santini committed
80

81
    void load(QSettings* settings)
Thiago Santini's avatar
Thiago Santini committed
82 83 84 85 86 87
    {
        settings->sync();
        set(settings, "width", inputSize.width);
        set(settings, "height", inputSize.height);
        set(settings, "flip", flip);
        set(settings, "undistort", undistort);
88 89
        set(settings, "coarseDetection", coarseDetection);
        set(settings, "processingDownscalingFactor", processingDownscalingFactor);
Thiago Santini's avatar
Thiago Santini committed
90
        set(settings, "pupilDetectionMethod", pupilDetectionMethod);
91 92
        set(settings, "tracking", tracking);
    }
Thiago Santini's avatar
Thiago Santini committed
93 94
};

95
class EyeImageProcessorUI : public QDialog {
Thiago Santini's avatar
Thiago Santini committed
96 97 98
    Q_OBJECT

public:
99
    EyeImageProcessorUI(QWidget* parent = 0)
Thiago Santini's avatar
Thiago Santini committed
100 101 102 103
        : QDialog(parent)
    {
        this->setWindowModality(Qt::ApplicationModal);
        this->setWindowTitle("Eye Image Processor Options");
104
        QGridLayout* layout = new QGridLayout();
Thiago Santini's avatar
Thiago Santini committed
105

106 107 108
        QHBoxLayout* hBoxLayout;
        QFormLayout* formLayout;
        QGroupBox* box;
Thiago Santini's avatar
Thiago Santini committed
109

110
        // TODO: make sure that the tooltip/whatsthis things are set for the QLabels as well
Thiago Santini's avatar
Thiago Santini committed
111 112 113 114 115 116 117
        box = new QGroupBox("Preprocessing");
        formLayout = new QFormLayout();
        widthSB = new QSpinBox();
        widthSB->setMaximum(4000);
        widthSB->setSuffix(" px");
        widthSB->setWhatsThis("Resizes the input width.\nThe resulting video will have this resolution.\nSetting to 0 disables resizing.");
        widthSB->setToolTip(box->whatsThis());
118
        formLayout->addRow(new QLabel("Width:"), widthSB);
Thiago Santini's avatar
Thiago Santini committed
119 120 121 122 123
        heightSB = new QSpinBox();
        heightSB->setMaximum(4000);
        heightSB->setSuffix(" px");
        heightSB->setWhatsThis("Resizes the input height.\nThe resulting video will have this resolution.\nSetting to 0 disables resizing.");
        heightSB->setToolTip(box->whatsThis());
124
        formLayout->addRow(new QLabel("Height:"), heightSB);
Thiago Santini's avatar
Thiago Santini committed
125 126 127 128 129 130 131 132 133 134 135 136
        flipComboBox = new QComboBox();
        flipComboBox->addItem("None", CV_FLIP_NONE);
        flipComboBox->addItem("Horizontal", CV_FLIP_HORIZONTAL);
        flipComboBox->addItem("Vertical", CV_FLIP_VERTICAL);
        flipComboBox->addItem("Both", CV_FLIP_BOTH);
        flipComboBox->setWhatsThis("Flips the input image.");
        flipComboBox->setToolTip(box->whatsThis());
        formLayout->addRow(new QLabel("Flip:"), flipComboBox);
        undistortBox = new QCheckBox();
        undistortBox->setWhatsThis("Undistorsts the input image.\nThe resulting video will be undistorted.\nNot recommended unless using homography gaze estimation.");
        undistortBox->setToolTip(box->whatsThis());
        undistortBox->setEnabled(false);
137
        formLayout->addRow(new QLabel("Undistort:"), undistortBox);
Thiago Santini's avatar
Thiago Santini committed
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
        box->setLayout(formLayout);
        layout->addWidget(box);

        downscalingSB = new QDoubleSpinBox();
        downscalingSB->setMinimum(1);
        downscalingSB->setSingleStep(1);
        hBoxLayout = new QHBoxLayout();
        box = new QGroupBox("Processing");
        box->setWhatsThis("Processing settings.\nNote: won't affect the recorded video (e.g., downscaling).");
        box->setToolTip(box->whatsThis());
        box->setLayout(hBoxLayout);
        hBoxLayout->addWidget(new QLabel("Downscaling Factor:"));
        hBoxLayout->addWidget(downscalingSB);
        layout->addWidget(box);

153 154 155
        formLayout = new QFormLayout();
        box = new QGroupBox("Pupil Detection");
        box->setWhatsThis("Selects pupil detection method.");
Thiago Santini's avatar
Thiago Santini committed
156
        box->setToolTip(box->whatsThis());
157 158 159 160 161 162 163 164 165 166 167 168
        box->setLayout(formLayout);
        coarseDetectionBox = new QCheckBox();
        coarseDetectionBox->setWhatsThis("Estimate a coarse location for the pupil location prior to detection.");
        coarseDetectionBox->setToolTip(box->whatsThis());
        formLayout->addRow(new QLabel("Coarse Detection:"), coarseDetectionBox);
        pupilDetectionComboBox = new QComboBox();
        formLayout->addRow(pupilDetectionComboBox);
        trackingBox = new QCheckBox();
        trackingBox->setWhatsThis("Track the pupil after detection using PuReST.");
        trackingBox->setToolTip(box->whatsThis());
        formLayout->addRow(new QLabel("PuReST (Santini et al. 2018b):"), trackingBox);
        layout->addWidget(box);
Thiago Santini's avatar
Thiago Santini committed
169 170 171 172 173 174 175

        applyButton = new QPushButton("Apply");
        applyButton->setWhatsThis("Applies current configuration.");
        applyButton->setToolTip(applyButton->whatsThis());
        layout->addWidget(applyButton);
        setLayout(layout);
        connect(applyButton, SIGNAL(clicked(bool)),
176
            this, SLOT(applyConfig()));
Thiago Santini's avatar
Thiago Santini committed
177
    }
178 179
    QSettings* settings;
    QComboBox* pupilDetectionComboBox;
Thiago Santini's avatar
Thiago Santini committed
180 181

signals:
182
    void updateConfig();
Thiago Santini's avatar
Thiago Santini committed
183 184 185 186 187 188 189 190 191

public slots:
    void showOptions(QPoint pos)
    {
        EyeImageProcessorConfig cfg;
        cfg.load(settings);

        widthSB->setValue(cfg.inputSize.width);
        heightSB->setValue(cfg.inputSize.height);
192 193 194
        downscalingSB->setValue(cfg.processingDownscalingFactor);
        coarseDetectionBox->setChecked(cfg.coarseDetection);
        for (int i = 0; i < flipComboBox->count(); i++)
Thiago Santini's avatar
Thiago Santini committed
195 196
            if (flipComboBox->itemData(i).toInt() == cfg.flip)
                flipComboBox->setCurrentIndex(i);
197
        for (int i = 0; i < pupilDetectionComboBox->count(); i++)
Thiago Santini's avatar
Thiago Santini committed
198 199
            if (pupilDetectionComboBox->itemData(i).toString() == cfg.pupilDetectionMethod)
                pupilDetectionComboBox->setCurrentIndex(i);
200 201
        trackingBox->setChecked(cfg.tracking);
        move(pos);
Thiago Santini's avatar
Thiago Santini committed
202 203 204 205 206 207 208
        show();
    }
    void applyConfig()
    {
        EyeImageProcessorConfig cfg;
        cfg.inputSize.width = widthSB->value();
        cfg.inputSize.height = heightSB->value();
209 210 211
        cfg.processingDownscalingFactor = downscalingSB->value();
        cfg.flip = static_cast<CVFlip>(flipComboBox->currentData().toInt());
        cfg.coarseDetection = coarseDetectionBox->isChecked();
Thiago Santini's avatar
Thiago Santini committed
212
        cfg.pupilDetectionMethod = pupilDetectionComboBox->currentData().toString();
213 214
        cfg.tracking = trackingBox->isChecked();
        cfg.save(settings);
Thiago Santini's avatar
Thiago Santini committed
215
        emit updateConfig();
216
    }
Thiago Santini's avatar
Thiago Santini committed
217 218

private:
219
    QPushButton* applyButton;
Thiago Santini's avatar
Thiago Santini committed
220
    QSpinBox *widthSB, *heightSB;
221 222 223 224 225
    QCheckBox* undistortBox;
    QCheckBox* coarseDetectionBox;
    QComboBox* flipComboBox;
    QDoubleSpinBox* downscalingSB;
    QCheckBox* trackingBox;
Thiago Santini's avatar
Thiago Santini committed
226 227
};

228
class EyeImageProcessor : public QObject {
Thiago Santini's avatar
Thiago Santini committed
229 230
    Q_OBJECT
public:
231
    explicit EyeImageProcessor(QString id, QObject* parent = 0);
Thiago Santini's avatar
Thiago Santini committed
232
    ~EyeImageProcessor();
233 234 235 236 237
    QSettings* settings;
    QVector<std::shared_ptr<PupilDetectionMethod>> availablePupilDetectionMethods;
    EyeImageProcessorConfig cfg;
    EyeData data;
    std::shared_ptr<CameraCalibration> cameraCalibration;
Thiago Santini's avatar
Thiago Santini committed
238 239

signals:
240
    void newData(EyeData data);
Thiago Santini's avatar
Thiago Santini committed
241 242

public slots:
243 244 245
    void process(Timestamp t, const cv::Mat& frame);
    void updateConfig();
    void newROI(QPointF sROI, QPointF eROI);
246
    void newMaxRadius(double maxRadius);
Thiago Santini's avatar
Thiago Santini committed
247 248 249

private:
    QString id;
250
    QMutex cfgMutex;
Thiago Santini's avatar
Thiago Santini committed
251

252 253
    std::shared_ptr<PupilDetectionMethod> pupilDetectionMethod;
    std::shared_ptr<PupilTrackingMethod> pupilTrackingMethod;
254

255
    unsigned int pmIdx;
Thiago Santini's avatar
Thiago Santini committed
256 257 258
};

#endif // EYEIMAGEPROCESSOR_H