FrameGrabber.cpp 6.44 KB
Newer Older
Thiago Santini's avatar
Thiago Santini committed
1 2
#include "FrameGrabber.h"

3 4
#include "globals.h"

Thiago Santini's avatar
Thiago Santini committed
5 6 7
using namespace std;
using namespace cv;

8 9 10 11 12 13 14 15
FrameGrabber::FrameGrabber(QString id, int code, QObject* parent)
    : QAbstractVideoSurface(parent)
    , id(id)
    , code(code)
    , yuvBuffer(nullptr)
    , yuvBufferSize(0)
    , timestampOffset(0)
    , timeoutMs(2e3)
Thiago Santini's avatar
Thiago Santini committed
16 17 18 19
{
#ifdef TURBOJPEG
    tjh = tjInitDecompress();
#endif
20

Thiago Santini's avatar
Thiago Santini committed
21 22
    watchdog = new QTimer(this);
    connect(watchdog, SIGNAL(timeout()), this, SIGNAL(timedout()));
23 24 25

    // for the first frame, we give a bit of extra leeway becase gstreamer is slow...
    //watchdog->start(timeoutMs);
26
    watchdog->start(15e3);
27 28

    pmIdx = gPerformanceMonitor.enrol(id, "Frame Grabber");
Thiago Santini's avatar
Thiago Santini committed
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
}

FrameGrabber::~FrameGrabber()
{
    watchdog->stop();
    watchdog->deleteLater();
#ifdef TURBOJPEG
    tjDestroy(tjh);
#endif
}

QList<QVideoFrame::PixelFormat> FrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const
{
    Q_UNUSED(handleType);
    return QList<QVideoFrame::PixelFormat>()
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
        << QVideoFrame::Format_Jpeg
        << QVideoFrame::Format_RGB32
        << QVideoFrame::Format_YUYV
        << QVideoFrame::Format_RGB24;
    //<< QVideoFrame::Format_ARGB32_Premultiplied
    //<< QVideoFrame::Format_RGB565
    //<< QVideoFrame::Format_RGB555
    //<< QVideoFrame::Format_ARGB8565_Premultiplied
    //<< QVideoFrame::Format_BGRA32
    //<< QVideoFrame::Format_BGRA32_Premultiplied
    //<< QVideoFrame::Format_BGR24
    //<< QVideoFrame::Format_BGR565
    //<< QVideoFrame::Format_BGR555
    //<< QVideoFrame::Format_BGRA5658_Premultiplied
    //<< QVideoFrame::Format_AYUV444
    //<< QVideoFrame::Format_AYUV444_Premultiplied
    //<< QVideoFrame::Format_YUV444
    //<< QVideoFrame::Format_YUV420P
    //<< QVideoFrame::Format_YV12
    //<< QVideoFrame::Format_UYVY
    //<< QVideoFrame::Format_YUYV
    //<< QVideoFrame::Format_NV12
    //<< QVideoFrame::Format_NV21
    //<< QVideoFrame::Format_IMC1
    //<< QVideoFrame::Format_IMC2
    //<< QVideoFrame::Format_IMC3
    //<< QVideoFrame::Format_IMC4
    //<< QVideoFrame::Format_Y8
    //<< QVideoFrame::Format_Y16
    //<< QVideoFrame::Format_CameraRaw
    //<< QVideoFrame::Format_AdobeDng;
Thiago Santini's avatar
Thiago Santini committed
75 76
}

77
bool FrameGrabber::present(const QVideoFrame& frame)
Thiago Santini's avatar
Thiago Santini committed
78 79 80 81 82 83 84 85 86 87
{
    /*
     * IMPORTANT:
     *
     * This frame's data lifetime is not guaranteed once we leave this function
     * so it shouldn't be used outside.
     * If sending the data somewhere else (e.g., for the preview) is necessary,
     * we must copy the data
     *
     */
88 89
    Timestamp t = ns2ms(gTimer.nsecsElapsed());

90 91
    /* TODO: Disabled for the time being; the number of samples here should be
     * better adjusted otherwise we risk getting the offset wrong
92 93 94 95 96 97 98 99 100 101 102
    const QString tMetaStr = "timestamp";
    if (frame.metaData(tMetaStr).isValid()) {
        Timestamp ft = qvariant_cast<Timestamp>(frame.metaData(tMetaStr));
        if (timestampOffsetEstimators.size() < 30) {
            // Use median of first N frames to estimate the offset
            timestampOffsetEstimators.emplace_back(t - ft);
            std::sort(timestampOffsetEstimators.begin(), timestampOffsetEstimators.end());
            timestampOffset = timestampOffsetEstimators[0.5 * (timestampOffsetEstimators.size() - 1)];
        }
        t = ft + timestampOffset;
    }
103
    */
Thiago Santini's avatar
Thiago Santini committed
104 105 106 107 108 109 110

    if (!frame.isValid())
        return false;

    QVideoFrame copy(frame);
    Mat cvFrame;

111
    copy.map(QAbstractVideoBuffer::ReadOnly);
Thiago Santini's avatar
Thiago Santini committed
112 113
    bool success = false;
    switch (frame.pixelFormat()) {
114 115 116 117 118 119 120 121 122 123 124 125
    case QVideoFrame::Format_Jpeg:
        success = jpeg2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_RGB32:
        success = rgb32_2bmp(copy, cvFrame);
        break;
    case QVideoFrame::Format_YUYV:
        success = yuyv_2bmp(copy, cvFrame);
        break;
    default:
        qWarning() << "Unknown pixel format:" << frame.pixelFormat();
        break;
Thiago Santini's avatar
Thiago Santini committed
126 127 128 129
    }
    copy.unmap();

    if (success && !cvFrame.empty()) {
130 131
        watchdog->start(timeoutMs);
        emit newFrame(t, cvFrame);
132 133
    } else
        gPerformanceMonitor.account(pmIdx);
Thiago Santini's avatar
Thiago Santini committed
134 135 136 137 138 139 140 141 142

    return success;
}

void FrameGrabber::setColorCode(int code)
{
    this->code = code;
}

143
bool FrameGrabber::jpeg2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
144
{
145
    unsigned char* frame = const_cast<unsigned char*>(in.bits());
Thiago Santini's avatar
Thiago Santini committed
146 147 148 149
    int len = in.mappedBytes();

#ifdef TURBOJPEG
    int width, height, subsamp, res;
150 151
    res = tjDecompressHeader2(tjh, frame, len, &width, &height, &subsamp);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
152 153 154 155 156
        qWarning() << QString("Frame drop; invalid header: ").append(tjGetErrorStr());
        return false;
    }

    long unsigned int bufSize = tjBufSizeYUV(width, height, subsamp);
157
    if (bufSize != yuvBufferSize) {
Thiago Santini's avatar
Thiago Santini committed
158 159 160 161 162 163
        //qInfo() << "YUV buffer size changed";
        yuvBufferSize = bufSize;
        delete yuvBuffer;
        yuvBuffer = new unsigned char[yuvBufferSize];
    }

164 165
    res = tjDecompressToYUV(tjh, frame, len, yuvBuffer, 0);
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
166 167 168 169
        qWarning() << QString("Frame drop; failed to decompress: ").append(tjGetErrorStr());
        return false;
    }

170
    cvFrame = Mat(height, width, code);
Thiago Santini's avatar
Thiago Santini committed
171 172
    int decode = code == CV_8UC3 ? TJPF_BGR : TJPF_GRAY;
    res = tjDecodeYUV(tjh, yuvBuffer, 4, subsamp, cvFrame.data, width, 0, height, decode, 0);
173
    if (res < 0) {
Thiago Santini's avatar
Thiago Santini committed
174 175 176 177
        qWarning() << QString("Frame drop; failed to decode: ").append(tjGetErrorStr());
        return false;
    }
#else
178
    std::vector<char> data(frame, frame + len);
Thiago Santini's avatar
Thiago Santini committed
179 180 181 182 183 184 185 186 187
    if (code == CV_8U)
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_GRAYSCALE);
    else
        cvFrame = imdecode(Mat(data), CV_LOAD_IMAGE_COLOR);
#endif

    return true;
}

188
bool FrameGrabber::rgb32_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
Thiago Santini's avatar
Thiago Santini committed
189 190
{
    // why abs? Some cameras seem to report some negative frame sizes for DirectShow; I'm looking at you Grasshopper!
191
    Mat rgba = Mat(abs(in.height()), abs(in.width()), CV_8UC4, (void*)in.bits());
Thiago Santini's avatar
Thiago Santini committed
192 193 194 195 196 197 198
    if (code == CV_8UC3)
        cvtColor(rgba, cvFrame, CV_BGRA2BGR);
    else
        cvtColor(rgba, cvFrame, CV_BGRA2GRAY);
    return true;
}

199
bool FrameGrabber::yuyv_2bmp(const QVideoFrame& in, cv::Mat& cvFrame)
200
{
201 202 203 204 205 206 207 208
    // some of the cheaper cameras tend to mess up the data for the first frames
    if (abs(in.height()) * abs(in.width()) * 2 > in.mappedBytes())
        return false;
    Mat yuyv = Mat(abs(in.height()), abs(in.width()), CV_8UC2, (void*)in.bits());
    if (code == CV_8UC3)
        cvtColor(yuyv, cvFrame, CV_YUV2BGR_YUYV);
    else
        cvtColor(yuyv, cvFrame, CV_YUV2GRAY_YUYV);
209 210
    return true;
}
Thiago Santini's avatar
Thiago Santini committed
211

Thiago Santini's avatar
Thiago Santini committed
212
//TODO: add support for other frame formats