EyeImageProcessor.h 10.3 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#ifndef EYEIMAGEPROCESSOR_H
#define EYEIMAGEPROCESSOR_H

#include <QObject>
#include <QApplication>
#include <QDialog>
#include <QGridLayout>
#include <QLabel>
#include <QComboBox>
#include <QPushButton>
#include <QSpinBox>
#include <QSettings>
#include <QGroupBox>
#include <QCheckBox>
#include <QFormLayout>

#include <opencv/cv.h>

#include "InputWidget.h"

21
#include "pupil-detection/PuRe.h"
Thiago Santini's avatar
Thiago Santini committed
22 23 24 25 26 27 28 29 30 31
#include "pupil-detection/ElSe.h"
#include "pupil-detection/ExCuSe.h"
#ifdef STARBURST
#include "pupil-detection/Starburst.h"
#endif
#ifdef SWIRSKI
#include "pupil-detection/Swirski.h"
#endif
#include "pupil-detection/PupilDetectionMethod.h"

Thiago Santini's avatar
Thiago Santini committed
32
#include "pupil-tracking/PuReST.h"
33 34
#include "pupil-tracking/PupilTrackingMethod.h"

Thiago Santini's avatar
Thiago Santini committed
35 36 37 38 39
#include "utils.h"

class EyeData : public InputData {
public:
    explicit EyeData(){
40
        timestamp = 0;
Thiago Santini's avatar
Thiago Santini committed
41
        input = cv::Mat();
42
		pupil = Pupil();
Thiago Santini's avatar
Thiago Santini committed
43 44 45 46 47
        validPupil = false;
        processingTimestamp = 0;
    }

    cv::Mat input;
48 49 50
	Pupil pupil;
	bool validPupil;
	cv::Rect coarseROI;
Thiago Santini's avatar
Thiago Santini committed
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67

    // TODO: header, toQString, and the reading from file (see the Calibration class) should be unified
    // to avoid placing things in the wrong order / with the wrong string
    QString header(QString prefix = "") const {
        QString tmp;
        tmp.append(prefix + "timestamp");
        tmp.append(gDataSeparator);
        tmp.append(prefix + "pupil.x");
        tmp.append(gDataSeparator);
        tmp.append(prefix + "pupil.y");
        tmp.append(gDataSeparator);
        tmp.append(prefix + "pupil.width");
        tmp.append(gDataSeparator);
        tmp.append(prefix + "pupil.height");
        tmp.append(gDataSeparator);
        tmp.append(prefix + "pupil.angle");
        tmp.append(gDataSeparator);
68 69 70
		tmp.append(prefix + "pupil.confidence");
		tmp.append(gDataSeparator);
		tmp.append(prefix + "pupil.valid");
Thiago Santini's avatar
Thiago Santini committed
71
        tmp.append(gDataSeparator);
72
		tmp.append(prefix + "processingTime");
Thiago Santini's avatar
Thiago Santini committed
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
        tmp.append(gDataSeparator);
        return tmp;
    }

    QString toQString() const {
        QString tmp;
        tmp.append(QString::number(timestamp));
        tmp.append(gDataSeparator);
        tmp.append(QString::number(pupil.center.x));
        tmp.append(gDataSeparator);
        tmp.append(QString::number(pupil.center.y));
        tmp.append(gDataSeparator);
        tmp.append(QString::number(pupil.size.width));
        tmp.append(gDataSeparator);
        tmp.append(QString::number(pupil.size.height));
        tmp.append(gDataSeparator);
89 90 91
		tmp.append(QString::number(pupil.angle));
		tmp.append(gDataSeparator);
		tmp.append(QString::number(pupil.confidence));
Thiago Santini's avatar
Thiago Santini committed
92 93 94 95 96 97 98 99 100
        tmp.append(gDataSeparator);
        tmp.append(QString::number(validPupil));
        tmp.append(gDataSeparator);
        tmp.append(QString::number(processingTimestamp));
        tmp.append(gDataSeparator);
        return tmp;
    }
};

101
Q_DECLARE_METATYPE(EyeData);
Thiago Santini's avatar
Thiago Santini committed
102 103 104 105 106 107 108 109 110


class EyeImageProcessorConfig
{
public:
    EyeImageProcessorConfig()
        :
          inputSize(cv::Size(0, 0)),
          flip(CV_FLIP_NONE),
111
		  undistort(false),
112
		  coarseDetection(false),
113
		  processingDownscalingFactor(1),
114 115
		  pupilDetectionMethod(PuRe::desc.c_str()),
		  tracking(true)
Thiago Santini's avatar
Thiago Santini committed
116 117 118 119
    {}

    cv::Size inputSize;
    CVFlip flip;
120 121
	bool undistort;
	bool coarseDetection;
Thiago Santini's avatar
Thiago Santini committed
122
    double processingDownscalingFactor;
123 124
	QString pupilDetectionMethod;
	bool tracking;
Thiago Santini's avatar
Thiago Santini committed
125 126 127 128 129 130 131 132

    void save(QSettings *settings)
    {
        settings->sync();
        settings->setValue("width", inputSize.width);
        settings->setValue("height", inputSize.height);
        settings->setValue("flip", flip);
        settings->setValue("undistort", undistort);
133 134
		settings->setValue("coarseDetection", coarseDetection);
		settings->setValue("processingDownscalingFactor", processingDownscalingFactor);
Thiago Santini's avatar
Thiago Santini committed
135
        settings->setValue("pupilDetectionMethod", pupilDetectionMethod);
136 137
		settings->setValue("tracking", tracking);
	}
Thiago Santini's avatar
Thiago Santini committed
138 139 140 141 142 143 144 145

    void load(QSettings *settings)
    {
        settings->sync();
        set(settings, "width", inputSize.width);
        set(settings, "height", inputSize.height);
        set(settings, "flip", flip);
        set(settings, "undistort", undistort);
146 147
		set(settings, "coarseDetection", coarseDetection);
		set(settings, "processingDownscalingFactor", processingDownscalingFactor);
Thiago Santini's avatar
Thiago Santini committed
148
        set(settings, "pupilDetectionMethod", pupilDetectionMethod);
149 150
		set(settings, "tracking", tracking);
	}
Thiago Santini's avatar
Thiago Santini committed
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
};

class EyeImageProcessorUI : public QDialog
{
    Q_OBJECT

public:
    EyeImageProcessorUI(QWidget *parent=0)
        : QDialog(parent)
    {
        this->setWindowModality(Qt::ApplicationModal);
        this->setWindowTitle("Eye Image Processor Options");
        QGridLayout *layout = new QGridLayout();

        QHBoxLayout *hBoxLayout;
        QFormLayout *formLayout;
        QGroupBox *box;

Thiago Santini's avatar
Thiago Santini committed
169
		// TODO: make sure that the tooltip/whatsthis things are set for the QLabels as well
Thiago Santini's avatar
Thiago Santini committed
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
        box = new QGroupBox("Preprocessing");
        formLayout = new QFormLayout();
        widthSB = new QSpinBox();
        widthSB->setMaximum(4000);
        widthSB->setSuffix(" px");
        widthSB->setWhatsThis("Resizes the input width.\nThe resulting video will have this resolution.\nSetting to 0 disables resizing.");
        widthSB->setToolTip(box->whatsThis());
        formLayout->addRow( new QLabel("Width:"), widthSB );
        heightSB = new QSpinBox();
        heightSB->setMaximum(4000);
        heightSB->setSuffix(" px");
        heightSB->setWhatsThis("Resizes the input height.\nThe resulting video will have this resolution.\nSetting to 0 disables resizing.");
        heightSB->setToolTip(box->whatsThis());
        formLayout->addRow( new QLabel("Height:"), heightSB );
        flipComboBox = new QComboBox();
        flipComboBox->addItem("None", CV_FLIP_NONE);
        flipComboBox->addItem("Horizontal", CV_FLIP_HORIZONTAL);
        flipComboBox->addItem("Vertical", CV_FLIP_VERTICAL);
        flipComboBox->addItem("Both", CV_FLIP_BOTH);
        flipComboBox->setWhatsThis("Flips the input image.");
        flipComboBox->setToolTip(box->whatsThis());
        formLayout->addRow(new QLabel("Flip:"), flipComboBox);
        undistortBox = new QCheckBox();
        undistortBox->setWhatsThis("Undistorsts the input image.\nThe resulting video will be undistorted.\nNot recommended unless using homography gaze estimation.");
        undistortBox->setToolTip(box->whatsThis());
        undistortBox->setEnabled(false);
        formLayout->addRow( new QLabel("Undistort:"), undistortBox );
        box->setLayout(formLayout);
        layout->addWidget(box);

        downscalingSB = new QDoubleSpinBox();
        downscalingSB->setMinimum(1);
        downscalingSB->setSingleStep(1);
        hBoxLayout = new QHBoxLayout();
        box = new QGroupBox("Processing");
        box->setWhatsThis("Processing settings.\nNote: won't affect the recorded video (e.g., downscaling).");
        box->setToolTip(box->whatsThis());
        box->setLayout(hBoxLayout);
        hBoxLayout->addWidget(new QLabel("Downscaling Factor:"));
        hBoxLayout->addWidget(downscalingSB);
        layout->addWidget(box);


213 214 215
		formLayout = new QFormLayout();
		box = new QGroupBox("Pupil Detection");
		box->setWhatsThis("Selects pupil detection method.");
Thiago Santini's avatar
Thiago Santini committed
216
        box->setToolTip(box->whatsThis());
217 218 219 220 221 222 223
		box->setLayout(formLayout);
		coarseDetectionBox = new QCheckBox();
		coarseDetectionBox->setWhatsThis("Estimate a coarse location for the pupil location prior to detection.");
		coarseDetectionBox->setToolTip(box->whatsThis());
		formLayout->addRow( new QLabel("Coarse Detection:"), coarseDetectionBox );
		pupilDetectionComboBox = new QComboBox();
		formLayout->addRow(pupilDetectionComboBox);
224
		trackingBox = new QCheckBox();
Thiago Santini's avatar
Thiago Santini committed
225
		trackingBox->setWhatsThis("Track the pupil after detection using PuReST.");
226
		trackingBox->setToolTip(box->whatsThis());
Thiago Santini's avatar
Thiago Santini committed
227
		formLayout->addRow( new QLabel("PuReST (Santini et al. 2018b):"), trackingBox );
228
		layout->addWidget(box);
Thiago Santini's avatar
Thiago Santini committed
229 230 231 232 233 234 235 236 237 238 239 240 241

        applyButton = new QPushButton("Apply");
        applyButton->setWhatsThis("Applies current configuration.");
        applyButton->setToolTip(applyButton->whatsThis());
        layout->addWidget(applyButton);
        setLayout(layout);
        connect(applyButton, SIGNAL(clicked(bool)),
                this, SLOT(applyConfig()) );
    }
    QSettings *settings;
    QComboBox *pupilDetectionComboBox;

signals:
242
	void updateConfig();
Thiago Santini's avatar
Thiago Santini committed
243 244 245 246 247 248 249 250 251

public slots:
    void showOptions(QPoint pos)
    {
        EyeImageProcessorConfig cfg;
        cfg.load(settings);

        widthSB->setValue(cfg.inputSize.width);
        heightSB->setValue(cfg.inputSize.height);
252 253
		downscalingSB->setValue(cfg.processingDownscalingFactor);
		coarseDetectionBox->setChecked(cfg.coarseDetection);
Thiago Santini's avatar
Thiago Santini committed
254 255 256 257 258 259
        for (int i=0; i<flipComboBox->count(); i++)
            if (flipComboBox->itemData(i).toInt() == cfg.flip)
                flipComboBox->setCurrentIndex(i);
        for (int i=0; i<pupilDetectionComboBox->count(); i++)
            if (pupilDetectionComboBox->itemData(i).toString() == cfg.pupilDetectionMethod)
                pupilDetectionComboBox->setCurrentIndex(i);
260 261
		trackingBox->setChecked(cfg.tracking);
		move(pos);
Thiago Santini's avatar
Thiago Santini committed
262 263 264 265 266 267 268
        show();
    }
    void applyConfig()
    {
        EyeImageProcessorConfig cfg;
        cfg.inputSize.width = widthSB->value();
        cfg.inputSize.height = heightSB->value();
269 270 271
		cfg.processingDownscalingFactor = downscalingSB->value();
		cfg.flip = (CVFlip) flipComboBox->currentData().toInt();
		cfg.coarseDetection = coarseDetectionBox->isChecked();
Thiago Santini's avatar
Thiago Santini committed
272
        cfg.pupilDetectionMethod = pupilDetectionComboBox->currentData().toString();
273 274
		cfg.tracking = trackingBox->isChecked();
		cfg.save(settings);
Thiago Santini's avatar
Thiago Santini committed
275
        emit updateConfig();
276
	}
Thiago Santini's avatar
Thiago Santini committed
277 278 279 280

private:
    QPushButton *applyButton;
    QSpinBox *widthSB, *heightSB;
281 282 283 284
	QCheckBox *undistortBox;
	QCheckBox *coarseDetectionBox;
	QComboBox *flipComboBox;
	QDoubleSpinBox *downscalingSB;
285
	QCheckBox *trackingBox;
Thiago Santini's avatar
Thiago Santini committed
286 287 288 289 290 291 292 293 294 295
};

class EyeImageProcessor : public QObject
{
    Q_OBJECT
public:
    explicit EyeImageProcessor(QString id, QObject *parent = 0);
    ~EyeImageProcessor();
    QSettings *settings;
    QVector<PupilDetectionMethod*> availablePupilDetectionMethods;
296 297
	EyeImageProcessorConfig cfg;
	EyeData data;
Thiago Santini's avatar
Thiago Santini committed
298 299 300 301 302

signals:
    void newData(EyeData data);

public slots:
303
	void process(Timestamp t, const cv::Mat &frame);
304 305
	void updateConfig();
	void newROI(QPointF sROI, QPointF eROI);
Thiago Santini's avatar
Thiago Santini committed
306 307 308

private:
    QString id;
309
	QMutex cfgMutex;
Thiago Santini's avatar
Thiago Santini committed
310 311 312
    QPointF sROI, eROI;

    PupilDetectionMethod *pupilDetectionMethod;
313
	PupilTrackingMethod *pupilTrackingMethod;
314

315
	unsigned int pmIdx;
Thiago Santini's avatar
Thiago Santini committed
316 317 318
};

#endif // EYEIMAGEPROCESSOR_H