EyeImageProcessor.cpp 5.47 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3 4 5
#include "EyeImageProcessor.h"
#include <opencv2/highgui.hpp>

using namespace cv;

6 7 8 9 10 11
EyeImageProcessor::EyeImageProcessor(QString id, QObject* parent)
    : QObject(parent)
    , cameraCalibration(nullptr)
    , id(id)
    , pupilDetectionMethod(nullptr)
    , pupilTrackingMethod(nullptr)
Thiago Santini's avatar
Thiago Santini committed
12
{
13 14 15
    availablePupilDetectionMethods.push_back(std::make_shared<PuRe>());
    availablePupilDetectionMethods.push_back(std::make_shared<ElSe>());
    availablePupilDetectionMethods.push_back(std::make_shared<ExCuSe>());
16 17 18

    availablePupilTrackingMethods.push_back(std::make_shared<PuReST>());

Thiago Santini's avatar
Thiago Santini committed
19
#ifdef STARBURST
20
    availablePupilDetectionMethods.push_back(std::make_shared<Starburst>());
Thiago Santini's avatar
Thiago Santini committed
21 22
#endif
#ifdef SWIRSKI
23
    availablePupilDetectionMethods.push_back(std::make_shared<Swirski>());
Thiago Santini's avatar
Thiago Santini committed
24
#endif
25
    settings = new QSettings(gCfgDir + "/" + id + " Image Processor.ini", QSettings::IniFormat);
Thiago Santini's avatar
Thiago Santini committed
26
    updateConfig();
27

28
    pmIdx = gPerformanceMonitor.enrol(id, "Image Processor");
Thiago Santini's avatar
Thiago Santini committed
29 30 31 32 33 34 35
}

void EyeImageProcessor::updateConfig()
{
    QMutexLocker locker(&cfgMutex);
    cfg.load(settings);

Thiago Santini's avatar
Thiago Santini committed
36
    pupilDetectionMethod = nullptr;
37 38 39 40 41 42 43 44
    for (auto method : availablePupilDetectionMethods)
        if (cfg.pupilDetectionMethod == QString(method->description().c_str()))
            pupilDetectionMethod = method;

    pupilTrackingMethod = nullptr;
    for (auto method : availablePupilTrackingMethods)
        if (cfg.pupilTrackingMethod == QString(method->description().c_str()))
            pupilTrackingMethod = method;
Thiago Santini's avatar
Thiago Santini committed
45 46 47 48 49
}

EyeImageProcessor::~EyeImageProcessor()
{
    availablePupilDetectionMethods.clear();
50
    availablePupilTrackingMethods.clear();
Thiago Santini's avatar
Thiago Santini committed
51

52
    if (settings)
Thiago Santini's avatar
Thiago Santini committed
53 54 55
        settings->deleteLater();
}

56
void EyeImageProcessor::process(Timestamp timestamp, const Mat& frame)
Thiago Santini's avatar
Thiago Santini committed
57 58
{
    // TODO: parametrize frame drop due to lack of processing power
59 60
    if (gPerformanceMonitor.shouldDrop(pmIdx, timestamp, 100))
        return;
Thiago Santini's avatar
Thiago Santini committed
61

62
    QMutexLocker locker(&cfgMutex);
Thiago Santini's avatar
Thiago Santini committed
63
    Timestamp processingStart = gTimer.elapsed();
Thiago Santini's avatar
Thiago Santini committed
64 65 66

    data.timestamp = timestamp;

67 68 69 70 71 72 73 74 75 76 77 78
    Q_ASSERT_X(frame.data != data.input.data, "Eye Image Processing", "Previous and current input image matches!");
    if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
        data.input = Mat(cfg.inputSize, frame.type());
        resize(frame, data.input, cfg.inputSize);
    } else {
        data.input = frame;
    }

    if (cfg.flip != CV_FLIP_NONE)
        flip(data.input, data.input, cfg.flip);

    if (data.input.channels() > 1) // TODO: make it algorithm dependent
79
        cvtColor(data.input, data.input, cv::COLOR_BGR2GRAY);
80

81 82
    float minPupilDiameterPx = cfg.minPupilDiameterRatio * data.input.rows;
    float maxPupilDiameterPx = cfg.maxPupilDiameterRatio * data.input.rows;
83 84 85 86

    data.pupil = Pupil();
    data.validPupil = false;
    if (pupilDetectionMethod != nullptr) {
87 88 89 90 91 92
        Rect userROI = {
            static_cast<int>(cfg.roi.x * data.input.cols),
            static_cast<int>(cfg.roi.y * data.input.rows),
            static_cast<int>(cfg.roi.width * data.input.cols),
            static_cast<int>(cfg.roi.height * data.input.rows)
        };
93 94 95 96 97 98

        float scalingFactor = 1;
        if (cfg.processingDownscalingFactor > 1)
            scalingFactor = static_cast<float>(1.0 / cfg.processingDownscalingFactor);

        /*
99 100
		 *  From here on, our reference frame is the scaled user ROI
		 */
101 102 103 104 105
        Mat downscaled;
        resize(data.input(userROI), downscaled, Size(),
            scalingFactor, scalingFactor,
            INTER_AREA);
        Rect coarseROI = { 0, 0, downscaled.cols, downscaled.rows };
106

107 108 109
        // Rescale pupil size limits as well
        minPupilDiameterPx *= scalingFactor;
        maxPupilDiameterPx *= scalingFactor;
110

111 112 113
        // If the user wants a coarse location and the method has none embedded,
        // we further constrain the search using the generic one
        if (!pupilDetectionMethod->hasCoarseLocation() && cfg.coarseDetection) {
114
            coarseROI = PupilDetectionMethod::coarsePupilDetection(downscaled, 0.5f, 60, 40);
115
            data.coarseROI = Rect(
116 117
                userROI.tl() + coarseROI.tl() / scalingFactor,
                userROI.tl() + coarseROI.br() / scalingFactor);
118 119 120
        } else
            data.coarseROI = Rect();

121
        if (pupilTrackingMethod)
122 123 124
            pupilTrackingMethod->detectAndTrack(timestamp, downscaled, coarseROI, data.pupil, pupilDetectionMethod, minPupilDiameterPx, maxPupilDiameterPx);
        else
            data.pupil = pupilDetectionMethod->detectWithConfidence(downscaled, coarseROI, minPupilDiameterPx, maxPupilDiameterPx);
Thiago Santini's avatar
Thiago Santini committed
125

126 127 128
        if (data.pupil.center.x > 0 && data.pupil.center.y > 0) {
            // Upscale
            data.pupil.resize(1.0f / scalingFactor);
129

130 131 132 133 134
            // User region shift
            data.pupil.shift(userROI.tl());
            data.validPupil = true;
        }
    }
Thiago Santini's avatar
Thiago Santini committed
135

136 137
    data.modelData = EyeModelData();

138
    data.cameraCalibration = cameraCalibration;
Thiago Santini's avatar
Thiago Santini committed
139
    data.processingTimestamp = gTimer.elapsed() - processingStart;
Thiago Santini's avatar
Thiago Santini committed
140 141 142 143 144 145 146 147

    emit newData(data);
}

void EyeImageProcessor::newROI(QPointF sROI, QPointF eROI)
{
    QMutexLocker locker(&cfgMutex);
    if (sROI.isNull() || eROI.isNull()) {
148
        cfg.roi = { 0.0f, 0.0f, 1.0f, 1.0f };
Thiago Santini's avatar
Thiago Santini committed
149
    } else {
150 151 152 153 154 155
        cfg.roi = {
            static_cast<float>(min(sROI.x(), eROI.x())),
            static_cast<float>(min(sROI.y(), eROI.y())),
            static_cast<float>(abs(eROI.x() - sROI.x())),
            static_cast<float>(abs(eROI.y() - sROI.y()))
        };
Thiago Santini's avatar
Thiago Santini committed
156 157
    }
}
158 159 160 161 162

void EyeImageProcessor::newMaxRadius(double maxRadius)
{
    cfg.maxPupilDiameterRatio = 2 * maxRadius;
}