PuRe.h 7.91 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
/*
 * Copyright (c) 2018, Thiago Santini
 *
 * Permission to use, copy, modify, and distribute this software and its
 * documentation for non-commercial purposes, without fee, and without a written
 * agreement is hereby granted, provided that:
 *
 * 1) the above copyright notice, this permission notice, and the subsequent
 * bibliographic references be included in all copies or substantial portions of
 * the software
 *
 * 2) the appropriate bibliographic references be made on related publications
 *
 * In this context, non-commercial means not intended for use towards commercial
 * advantage (e.g., as complement to or part of a product) or monetary
 * compensation. The copyright holder reserves the right to decide whether a
 * certain use classifies as commercial or not. For commercial use, please contact
 * the copyright holders.
 *
 * REFERENCES:
 *
 * Thiago Santini, Wolfgang Fuhl, Enkelejda Kasneci, PuRe: Robust pupil detection
 * for real-time pervasive eye tracking, Computer Vision and Image Understanding,
 * 2018, ISSN 1077-3142, https://doi.org/10.1016/j.cviu.2018.02.002.
 *
 *
 * IN NO EVENT SHALL THE AUTHORS BE LIABLE TO ANY PARTY FOR DIRECT,
 * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
 * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
 * THE AUTHORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * THE AUTHORS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE AUTHORS
 * HAVE NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
 * MODIFICATIONS.
 */

#ifndef PURE_H
#define PURE_H

#include <bitset>
#include <random>

#include <QDebug>
46
#include <QString>
Thiago Santini's avatar
Thiago Santini committed
47 48

#include <opencv2/highgui.hpp>
49
#include <opencv2/imgproc/imgproc.hpp>
Thiago Santini's avatar
Thiago Santini committed
50 51 52 53
#include <opencv2/opencv.hpp>

#include "PupilDetectionMethod.h"

54
class PupilCandidate {
Thiago Santini's avatar
Thiago Santini committed
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
public:
    std::vector<cv::Point> points;
    cv::RotatedRect pointsMinAreaRect;
    float minCurvatureRatio;

    cv::RotatedRect outline;

    cv::Rect pointsBoundingBox;
    cv::Rect combinationRegion;
    cv::Rect br;
    cv::Rect boundaries;
    cv::Point2f v[4];
    cv::Rect outlineInscribedRect;
    cv::Point2f mp;
    float minorAxis, majorAxis;
    float aspectRatio;
    cv::Mat internalArea;
    float innerMeanIntensity;
    float outerMeanIntensity;
    float contrast;
    float outlineContrast;
    float anchorDistribution;
    float score;
78
    std::bitset<4> anchorPointSlices;
Thiago Santini's avatar
Thiago Santini committed
79

80
    cv::Scalar color;
Thiago Santini's avatar
Thiago Santini committed
81 82 83 84 85 86 87 88

    enum {
        Q0 = 0,
        Q1 = 1,
        Q2 = 2,
        Q3 = 3,
    };

89 90 91 92 93 94 95 96
    PupilCandidate(std::vector<cv::Point> points)
        : minCurvatureRatio(0.198912f)
        , // (1-cos(22.5))/sin(22.5)
        aspectRatio(0.0f)
        , outlineContrast(0.0f)
        , anchorDistribution(0.0f)
        , score(0.0f)
        , color(0, 255, 0)
Thiago Santini's avatar
Thiago Santini committed
97 98 99
    {
        this->points = points;
    }
100
    bool isValid(const cv::Mat& intensityImage, const int& minPupilDiameterPx, const int& maxPupilDiameterPx, const int bias = 5);
Thiago Santini's avatar
Thiago Santini committed
101 102 103 104
    void estimateOutline();
    bool isCurvatureValid();

    // Support functions
105 106 107
    float ratio(float a, float b)
    {
        std::pair<float, float> sorted = std::minmax(a, b);
Thiago Santini's avatar
Thiago Santini committed
108 109 110
        return sorted.first / sorted.second;
    }

111
    bool operator<(const PupilCandidate& c) const
Thiago Santini's avatar
Thiago Santini committed
112 113 114 115
    {
        return (score < c.score);
    }

116
    bool fastValidityCheck(const int& maxPupilDiameterPx);
Thiago Santini's avatar
Thiago Santini committed
117 118 119

    bool validateAnchorDistribution();

120
    bool validityCheck(const cv::Mat& intensityImage, const int& bias);
Thiago Santini's avatar
Thiago Santini committed
121

122 123
    bool validateOutlineContrast(const cv::Mat& intensityImage, const int& bias);
    bool drawOutlineContrast(const cv::Mat& intensityImage, const int& bias, QString out);
Thiago Santini's avatar
Thiago Santini committed
124 125 126

    void updateScore()
    {
127
        score = 0.33f * aspectRatio + 0.33f * anchorDistribution + 0.34f * outlineContrast;
Thiago Santini's avatar
Thiago Santini committed
128 129 130 131
        // ElSe style
        //score = (1-innerMeanIntensity)*(1+abs(outline.size.height-outline.size.width));
    }

132 133
    void draw(cv::Mat out)
    {
Thiago Santini's avatar
Thiago Santini committed
134 135
        //cv::ellipse(out, outline, cv::Scalar(0,255,0));
        //cv::rectangle(out, combinationRegion, cv::Scalar(0,255,255));
136
        for (unsigned int i = 0; i < points.size(); i++)
Thiago Santini's avatar
Thiago Santini committed
137 138 139
            cv::circle(out, points[i], 1, cv::Scalar(0, 255, 255));

        cv::circle(out, mp, 3, cv::Scalar(0, 0, 255), -1);
140
        QString s = QString("%1 %2 %3").arg(QString::number(aspectRatio, 'g', 2)).arg(QString::number(anchorDistribution, 'g', 2)).arg(QString::number(contrast, 'g', 2));
Thiago Santini's avatar
Thiago Santini committed
141 142 143 144 145
        //cv::putText(out, s.toStdString(), outline.center, CV_FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,255));
        //cv::putText(out, QString::number(score,'g', 2).toStdString(), outline.center, CV_FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,255));
        //cv::putText(out, QString::number(anchorDistribution,'g', 2).toStdString(), outline.center, CV_FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,255));
    }

146 147 148
    void draw(cv::Mat out, cv::Scalar color)
    {
        int w = 2;
Thiago Santini's avatar
Thiago Santini committed
149
        cv::circle(out, points[0], w, color, -1);
150
        for (unsigned int i = 1; i < points.size(); i++) {
Thiago Santini's avatar
Thiago Santini committed
151
            cv::circle(out, points[i], w, color, -1);
152
            cv::line(out, points[i - 1], points[i], color, w - 1);
Thiago Santini's avatar
Thiago Santini committed
153
        }
154 155
        cv::line(out, points[points.size() - 1], points[0], color, w - 1);
    }
Thiago Santini's avatar
Thiago Santini committed
156

157 158 159 160 161 162 163
    void drawit(cv::Mat out, cv::Scalar color)
    {
        int w = 2;
        for (unsigned int i = 0; i < points.size(); i++)
            cv::circle(out, points[i], w, color, -1);
        cv::ellipse(out, outline, color);
    }
Thiago Santini's avatar
Thiago Santini committed
164 165
};

166
class PuRe : public PupilDetectionMethod {
Thiago Santini's avatar
Thiago Santini committed
167 168 169 170
public:
    PuRe();
    ~PuRe();

171 172 173 174
    Pupil implDetect(const cv::Mat& frame, cv::Rect roi, const float& userMinPupilDiameterPx, const float& userMaxPupilDiameterPx) override;
    bool hasConfidence() override { return true; }
    bool hasCoarseLocation() override { return false; }
    static std::string desc;
Thiago Santini's avatar
Thiago Santini committed
175

176 177 178
    float meanCanthiDistanceMM;
    float maxPupilDiameterMM;
    float minPupilDiameterMM;
Thiago Santini's avatar
Thiago Santini committed
179

180 181 182
    float maxIrisDiameterMM;
    float meanIrisDiameterMM;
    float minIrisDiameterMM;
Thiago Santini's avatar
Thiago Santini committed
183 184 185 186 187 188 189 190 191 192 193 194

protected:
    cv::RotatedRect detectedPupil;
    cv::Size expectedFrameSize;

    int outlineBias;

    static const cv::RotatedRect invalidPupil;

    /*
     *  Initialization
     */
195 196
    void init(const cv::Mat& frame);
    void estimateParameters(const int rows, const int cols);
Thiago Santini's avatar
Thiago Santini committed
197 198 199 200 201 202

    /*
     * Downscaling
     */
    cv::Size baseSize;
    cv::Size workingSize;
203
    float scalingRatio;
Thiago Santini's avatar
Thiago Santini committed
204 205 206 207

    /*
     *  Detection
     */
208
    void detect(Pupil& pupil);
Thiago Santini's avatar
Thiago Santini committed
209 210

    // Canny
Thiago Santini's avatar
Thiago Santini committed
211
    cv::Mat blurred, dx, dy, magnitude;
Thiago Santini's avatar
Thiago Santini committed
212
    cv::Mat edgeType, edge;
213
    cv::Mat canny(const cv::Mat& in, const bool blur = true, const bool useL2 = true, const int bins = 64, const float nonEdgePixelsRatio = 0.7f, const float lowHighThresholdRatio = 0.4f);
Thiago Santini's avatar
Thiago Santini committed
214 215

    // Edge filtering
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
    void filterEdges(cv::Mat& edges);

    // Remove duplicates (e.g., from closed loops)
    int pointHash(const cv::Point& p, const int cols) { return p.y * cols + p.x; }
    void removeDuplicates(std::vector<std::vector<cv::Point>>& curves, const int& cols)
    {
        std::map<int, uchar> contourMap;
        for (size_t i = curves.size(); i-- > 0;) {
            if (contourMap.count(pointHash(curves[i][0], cols)) > 0)
                curves.erase(curves.begin() + i);
            else {
                for (int j = 0; j < curves[i].size(); j++)
                    contourMap[pointHash(curves[i][j], cols)] = 1;
            }
        }
    }

    void findPupilEdgeCandidates(const cv::Mat& intensityImage, cv::Mat& edge, std::vector<PupilCandidate>& candidates);
    void combineEdgeCandidates(const cv::Mat& intensityImage, cv::Mat& edge, std::vector<PupilCandidate>& candidates);
    void searchInnerCandidates(std::vector<PupilCandidate>& candidates, PupilCandidate& candidate);
Thiago Santini's avatar
Thiago Santini committed
236 237 238 239 240 241 242 243 244 245 246

    cv::Mat input;
    cv::Mat dbg;

    int maxCanthiDistancePx;
    int minCanthiDistancePx;
    int maxPupilDiameterPx;
    int minPupilDiameterPx;
};

#endif // PURE_H