FrameGrabber.cpp 6.65 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2
#include "FrameGrabber.h"

3 4
#include "globals.h"

Thiago Santini's avatar
Thiago Santini committed
5 6 7
using namespace std;
using namespace cv;

8 9 10 11 12 13 14
FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
    : QAbstractVideoSurface(parent)
    , id(id)
    , code(code)
    , yuvBuffer(nullptr)
    , yuvBufferSize(0)
    , timeoutMs(2e3)
Thiago Santini's avatar
Thiago Santini committed
15 16 17 18
{
#ifdef TURBOJPEG
    tjh = tjInitDecompress();
#endif
19

Thiago Santini's avatar
Thiago Santini committed
20 21
    watchdog = new QTimer(this);
    connect(watchdog, SIGNAL(timeout()), this, SIGNAL(timedout()));
22 23 24

    // for the first frame, we give a bit of extra leeway becase gstreamer is slow...
    //watchdog->start(timeoutMs);
25
    watchdog->start(15e3);
26 27

    pmIdx = gPerformanceMonitor.enrol(id, "Frame Grabber");
Thiago Santini's avatar
Thiago Santini committed
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
}

FrameGrabber::~FrameGrabber()
{
    watchdog->stop();
    watchdog->deleteLater();
#ifdef TURBOJPEG
    tjDestroy(tjh);
#endif
}

QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const
{
    Q_UNUSED(handleType);
    return QList<QVideoFrame::PixelFormat>()
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
        << QVideoFrame::Format_Jpeg
        << QVideoFrame::Format_RGB32
        << QVideoFrame::Format_YUYV
        << QVideoFrame::Format_RGB24;
    //<< QVideoFrame::Format_ARGB32_Premultiplied
    //<< QVideoFrame::Format_RGB565
    //<< QVideoFrame::Format_RGB555
    //<< QVideoFrame::Format_ARGB8565_Premultiplied
    //<< QVideoFrame::Format_BGRA32
    //<< QVideoFrame::Format_BGRA32_Premultiplied
    //<< QVideoFrame::Format_BGR24
    //<< QVideoFrame::Format_BGR565
    //<< QVideoFrame::Format_BGR555
    //<< QVideoFrame::Format_BGRA5658_Premultiplied
    //<< QVideoFrame::Format_AYUV444
    //<< QVideoFrame::Format_AYUV444_Premultiplied
    //<< QVideoFrame::Format_YUV444
    //<< QVideoFrame::Format_YUV420P
    //<< QVideoFrame::Format_YV12
    //<< QVideoFrame::Format_UYVY
    //<< QVideoFrame::Format_YUYV
    //<< QVideoFrame::Format_NV12
    //<< QVideoFrame::Format_NV21
    //<< QVideoFrame::Format_IMC1
    //<< QVideoFrame::Format_IMC2
    //<< QVideoFrame::Format_IMC3
    //<< QVideoFrame::Format_IMC4
    //<< QVideoFrame::Format_Y8
    //<< QVideoFrame::Format_Y16
    //<< QVideoFrame::Format_CameraRaw
    //<< QVideoFrame::Format_AdobeDng;
Thiago Santini's avatar
Thiago Santini committed
74 75
}

Thiago Santini's avatar
Thiago Santini committed
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
Timestamp FrameGrabber::getTimestamp(const QVideoFrame& frame)
{
    /* We have three options for timestamps:
     * 1 EyeRec software timestamp (t)
     * 2 Frame presentation timestamp (pts)
     * 3 Frame software timestamp (fts) -- at least for the uvcengine
     *
     * However, we can't guarante the source for pts and fts, so we need to keep
     * track of drift
     */
    auto ts = gTimer.elapsed();

    // 1
    auto selected = ts;

    if (frame.startTime() > 0 && frame.endTime() > 0) {
        // 2
        const auto pts = 1e-3 * 0.5 * (frame.endTime() + frame.startTime());
        selected = drift.correct(ts, pts);
    } else {
        // 3
        const auto& metaNow = frame.metaData("steady_clock::now");
        if (metaNow.isValid()) {
            const auto fts = gTimer.elapsed(qvariant_cast<SteadyTimePoint>(metaNow));
            selected = drift.correct(ts, fts);
        }
    }

    return selected;
}

107
bool FrameGrabber::present(const QVideoFrame& frame)
Thiago Santini's avatar
Thiago Santini committed
108 109 110 111 112 113 114 115 116 117
{
    /*
     * IMPORTANT:
     *
     * This frame's data lifetime is not guaranteed once we leave this function
     * so it shouldn't be used outside.
     * If sending the data somewhere else (e.g., for the preview) is necessary,
     * we must copy the data
     *
     */
Thiago Santini's avatar
Thiago Santini committed
118 119 120 121
    using namespace std::chrono;

    // Get SW timestamp asap
    auto t = getTimestamp(frame);
Thiago Santini's avatar
Thiago Santini committed
122 123 124 125 126 127 128

    if (!frame.isValid())
        return false;

    QVideoFrame copy(frame);
    Mat cvFrame;

129
    copy.map(QAbstractVideoBuffer::ReadOnly);
Thiago Santini's avatar
Thiago Santini committed
130 131
    bool success = false;
    switch (frame.pixelFormat()) {
132 133 134 135 136 137 138 139 140 141 142 143
    case QVideoFrame::Format_Jpeg:
        success = jpeg2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_RGB32:
        success = rgb32_2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_YUYV:
        success = yuyv_2bmp(copy, cvFrame);
        break;
    default:
        qWarning() << "Unknown pixel format:" << frame.pixelFormat();
        break;
Thiago Santini's avatar
Thiago Santini committed
144 145 146 147
    }
    copy.unmap();

    if (success && !cvFrame.empty()) {
148 149
        watchdog->start(timeoutMs);
        emit newFrame(t, cvFrame);
150 151
    } else
        gPerformanceMonitor.account(pmIdx);
Thiago Santini's avatar
Thiago Santini committed
152 153 154 155 156 157 158 159 160

    return success;
}

void FrameGrabber::setColorCode(int code)
{
    this->code = code;
}

161
bool FrameGrabber::jpeg2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
162
{
163
    unsigned char* frame = const_cast<unsigned char*>(in.bits());
Thiago Santini's avatar
Thiago Santini committed
164 165 166 167
    int len = in.mappedBytes();

#ifdef TURBOJPEG
    int width, height, subsamp, res;
168 169
    res = tjDecompressHeader2(tjh, frame, len, &width, &height, &subsamp);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
170 171 172 173 174
        qWarning() << QString("Frame drop; invalid header: ").append(tjGetErrorStr());
        return false;
    }

    long unsigned int bufSize = tjBufSizeYUV(width, height, subsamp);
175
    if (bufSize != yuvBufferSize) {
Thiago Santini's avatar
Thiago Santini committed
176 177 178 179 180 181
        //qInfo() << "YUV buffer size changed";
        yuvBufferSize = bufSize;
        delete yuvBuffer;
        yuvBuffer = new unsigned char[yuvBufferSize];
    }

182 183
    res = tjDecompressToYUV(tjh, frame, len, yuvBuffer, 0);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
184 185 186 187
        qWarning() << QString("Frame drop; failed to decompress: ").append(tjGetErrorStr());
        return false;
    }

188
    cvFrame = Mat(height, width, code);
Thiago Santini's avatar
Thiago Santini committed
189 190
    int decode = code == CV_8UC3 ? TJPF_BGR : TJPF_GRAY;
    res = tjDecodeYUV(tjh, yuvBuffer, 4, subsamp, cvFrame.data, width, 0, height, decode, 0);
191
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
192 193 194 195
        qWarning() << QString("Frame drop; failed to decode: ").append(tjGetErrorStr());
        return false;
    }
#else
196
    std::vector<char> data(frame, frame + len);
Thiago Santini's avatar
Thiago Santini committed
197 198 199 200 201 202 203 204 205
    if (code == CV_8U)
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_GRAYSCALE);
    else
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_COLOR);
#endif

    return true;
}

206
bool FrameGrabber::rgb32_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
207 208
{
    // why abs? Some cameras seem to report some negative frame sizes for DirectShow; I'm looking at you Grasshopper!
209
    Mat rgba = Mat(abs(in.height()), abs(in.width()), CV_8UC4, (void*)in.bits());
Thiago Santini's avatar
Thiago Santini committed
210 211 212 213 214 215 216
    if (code == CV_8UC3)
        cvtColor(rgba, cvFrame, CV_BGRA2BGR);
    else
        cvtColor(rgba, cvFrame, CV_BGRA2GRAY);
    return true;
}

217
bool FrameGrabber::yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
218
{
219 220 221 222 223 224 225 226
    // some of the cheaper cameras tend to mess up the data for the first frames
    if (abs(in.height()) * abs(in.width()) * 2 > in.mappedBytes())
        return false;
    Mat yuyv = Mat(abs(in.height()), abs(in.width()), CV_8UC2, (void*)in.bits());
    if (code == CV_8UC3)
        cvtColor(yuyv, cvFrame, CV_YUV2BGR_YUYV);
    else
        cvtColor(yuyv, cvFrame, CV_YUV2GRAY_YUYV);
227 228
    return true;
}
Thiago Santini's avatar
Thiago Santini committed
229

Thiago Santini's avatar
Thiago Santini committed
230
//TODO: add support for other frame formats