FrameGrabber.cpp 6.5 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2
#include "FrameGrabber.h"

3 4
#include "globals.h"

Thiago Santini's avatar
Thiago Santini committed
5 6 7
using namespace std;
using namespace cv;

8 9 10 11 12 13
FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
    : QAbstractVideoSurface(parent)
    , id(id)
    , code(code)
    , yuvBuffer(nullptr)
    , yuvBufferSize(0)
14
    , watchdog(new QTimer(this))
15
    , timeoutMs(2e3)
Thiago Santini's avatar
Thiago Santini committed
16 17 18 19
{
#ifdef TURBOJPEG
    tjh = tjInitDecompress();
#endif
20

Thiago Santini's avatar
Thiago Santini committed
21
    connect(watchdog, SIGNAL(timeout()), this, SIGNAL(timedout()));
22

23
    pmIdx = gPerformanceMonitor.enrol(id, "Frame Grabber");
Thiago Santini's avatar
Thiago Santini committed
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
}

FrameGrabber::~FrameGrabber()
{
    watchdog->stop();
    watchdog->deleteLater();
#ifdef TURBOJPEG
    tjDestroy(tjh);
#endif
}

QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const
{
    Q_UNUSED(handleType);
    return QList<QVideoFrame::PixelFormat>()
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
        << QVideoFrame::Format_Jpeg
        << QVideoFrame::Format_RGB32
        << QVideoFrame::Format_YUYV
        << QVideoFrame::Format_RGB24;
    //<< QVideoFrame::Format_ARGB32_Premultiplied
    //<< QVideoFrame::Format_RGB565
    //<< QVideoFrame::Format_RGB555
    //<< QVideoFrame::Format_ARGB8565_Premultiplied
    //<< QVideoFrame::Format_BGRA32
    //<< QVideoFrame::Format_BGRA32_Premultiplied
    //<< QVideoFrame::Format_BGR24
    //<< QVideoFrame::Format_BGR565
    //<< QVideoFrame::Format_BGR555
    //<< QVideoFrame::Format_BGRA5658_Premultiplied
    //<< QVideoFrame::Format_AYUV444
    //<< QVideoFrame::Format_AYUV444_Premultiplied
    //<< QVideoFrame::Format_YUV444
    //<< QVideoFrame::Format_YUV420P
    //<< QVideoFrame::Format_YV12
    //<< QVideoFrame::Format_UYVY
    //<< QVideoFrame::Format_YUYV
    //<< QVideoFrame::Format_NV12
    //<< QVideoFrame::Format_NV21
    //<< QVideoFrame::Format_IMC1
    //<< QVideoFrame::Format_IMC2
    //<< QVideoFrame::Format_IMC3
    //<< QVideoFrame::Format_IMC4
    //<< QVideoFrame::Format_Y8
    //<< QVideoFrame::Format_Y16
    //<< QVideoFrame::Format_CameraRaw
    //<< QVideoFrame::Format_AdobeDng;
Thiago Santini's avatar
Thiago Santini committed
70 71
}

Thiago Santini's avatar
Thiago Santini committed
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
Timestamp FrameGrabber::getTimestamp(const QVideoFrame& frame)
{
    /* We have three options for timestamps:
     * 1 EyeRec software timestamp (t)
     * 2 Frame presentation timestamp (pts)
     * 3 Frame software timestamp (fts) -- at least for the uvcengine
     *
     * However, we can't guarante the source for pts and fts, so we need to keep
     * track of drift
     */
    auto ts = gTimer.elapsed();

    // 1
    auto selected = ts;

    if (frame.startTime() > 0 && frame.endTime() > 0) {
        // 2
        const auto pts = 1e-3 * 0.5 * (frame.endTime() + frame.startTime());
        selected = drift.correct(ts, pts);
    } else {
        // 3
        const auto& metaNow = frame.metaData("steady_clock::now");
        if (metaNow.isValid()) {
            const auto fts = gTimer.elapsed(qvariant_cast<SteadyTimePoint>(metaNow));
            selected = drift.correct(ts, fts);
        }
    }

    return selected;
}

103
bool FrameGrabber::present(const QVideoFrame& frame)
Thiago Santini's avatar
Thiago Santini committed
104 105 106 107 108 109 110 111 112 113
{
    /*
     * IMPORTANT:
     *
     * This frame's data lifetime is not guaranteed once we leave this function
     * so it shouldn't be used outside.
     * If sending the data somewhere else (e.g., for the preview) is necessary,
     * we must copy the data
     *
     */
Thiago Santini's avatar
Thiago Santini committed
114 115 116 117
    using namespace std::chrono;

    // Get SW timestamp asap
    auto t = getTimestamp(frame);
Thiago Santini's avatar
Thiago Santini committed
118 119 120 121 122 123 124

    if (!frame.isValid())
        return false;

    QVideoFrame copy(frame);
    Mat cvFrame;

125
    copy.map(QAbstractVideoBuffer::ReadOnly);
Thiago Santini's avatar
Thiago Santini committed
126 127
    bool success = false;
    switch (frame.pixelFormat()) {
128 129 130 131 132 133 134 135 136 137 138 139
    case QVideoFrame::Format_Jpeg:
        success = jpeg2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_RGB32:
        success = rgb32_2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_YUYV:
        success = yuyv_2bmp(copy, cvFrame);
        break;
    default:
        qWarning() << "Unknown pixel format:" << frame.pixelFormat();
        break;
Thiago Santini's avatar
Thiago Santini committed
140 141 142 143
    }
    copy.unmap();

    if (success && !cvFrame.empty()) {
144 145
        watchdog->start(timeoutMs);
        emit newFrame(t, cvFrame);
146 147
    } else
        gPerformanceMonitor.account(pmIdx);
Thiago Santini's avatar
Thiago Santini committed
148 149 150 151 152 153 154 155 156

    return success;
}

void FrameGrabber::setColorCode(int code)
{
    this->code = code;
}

157
bool FrameGrabber::jpeg2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
158
{
159
    auto frame = const_cast<unsigned char*>(in.bits());
Thiago Santini's avatar
Thiago Santini committed
160 161 162 163
    int len = in.mappedBytes();

#ifdef TURBOJPEG
    int width, height, subsamp, res;
164 165
    res = tjDecompressHeader2(tjh, frame, len, &width, &height, &subsamp);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
166 167 168 169 170
        qWarning() << QString("Frame drop; invalid header: ").append(tjGetErrorStr());
        return false;
    }

    long unsigned int bufSize = tjBufSizeYUV(width, height, subsamp);
171
    if (bufSize != yuvBufferSize) {
Thiago Santini's avatar
Thiago Santini committed
172 173 174 175 176 177
        //qInfo() << "YUV buffer size changed";
        yuvBufferSize = bufSize;
        delete yuvBuffer;
        yuvBuffer = new unsigned char[yuvBufferSize];
    }

178 179
    res = tjDecompressToYUV(tjh, frame, len, yuvBuffer, 0);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
180 181 182 183
        qWarning() << QString("Frame drop; failed to decompress: ").append(tjGetErrorStr());
        return false;
    }

184
    cvFrame = Mat(height, width, code);
Thiago Santini's avatar
Thiago Santini committed
185 186
    int decode = code == CV_8UC3 ? TJPF_BGR : TJPF_GRAY;
    res = tjDecodeYUV(tjh, yuvBuffer, 4, subsamp, cvFrame.data, width, 0, height, decode, 0);
187
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
188 189 190 191
        qWarning() << QString("Frame drop; failed to decode: ").append(tjGetErrorStr());
        return false;
    }
#else
192
    std::vector<char> data(frame, frame + len);
Thiago Santini's avatar
Thiago Santini committed
193 194 195 196 197 198 199 200 201
    if (code == CV_8U)
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_GRAYSCALE);
    else
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_COLOR);
#endif

    return true;
}

202
bool FrameGrabber::rgb32_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
203 204
{
    // why abs? Some cameras seem to report some negative frame sizes for DirectShow; I'm looking at you Grasshopper!
205
    Mat rgba = Mat(abs(in.height()), abs(in.width()), CV_8UC4, (void*)in.bits());
Thiago Santini's avatar
Thiago Santini committed
206 207 208 209 210 211 212
    if (code == CV_8UC3)
        cvtColor(rgba, cvFrame, CV_BGRA2BGR);
    else
        cvtColor(rgba, cvFrame, CV_BGRA2GRAY);
    return true;
}

213
bool FrameGrabber::yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
214
{
215 216 217 218 219 220 221 222
    // some of the cheaper cameras tend to mess up the data for the first frames
    if (abs(in.height()) * abs(in.width()) * 2 > in.mappedBytes())
        return false;
    Mat yuyv = Mat(abs(in.height()), abs(in.width()), CV_8UC2, (void*)in.bits());
    if (code == CV_8UC3)
        cvtColor(yuyv, cvFrame, CV_YUV2BGR_YUYV);
    else
        cvtColor(yuyv, cvFrame, CV_YUV2GRAY_YUYV);
223 224
    return true;
}
Thiago Santini's avatar
Thiago Santini committed
225

Thiago Santini's avatar
Thiago Santini committed
226
//TODO: add support for other frame formats