Adds performance improvements

Code further profiled, and removed a couple unnecessary operations.

Added information about possible move to OpenGL for the camera widgets.

Fixed bug for eye processor pre-processing downscaling.

EyeRecToo/src/CameraWidget.cpp

@@ -8,7 +8,6 @@ CameraWidget::CameraWidget(QString id, ImageProcessor::Type type, QWidget *paren
     QMainWindow(parent),
     id(id),
     type(type),
-    lastTimestamp(0),
     sROI(QPoint(0,0)),
     eROI(QPoint(0,0)),
     settingROI(false),
@@ -122,7 +121,10 @@ CameraWidget::CameraWidget(QString id, ImageProcessor::Type type, QWidget *paren
     connect(this, SIGNAL(newROI(QPointF,QPointF)),
             imageProcessor, SIGNAL(newROI(QPointF,QPointF)) );
 
-    font.setStyleHint(QFont::Monospace);
+	// Initial roi
+	setROI( QPointF(0.1, 0.1), QPointF(0.9, 0.9) );
+
+	font.setStyleHint(QFont::Monospace);
     QMetaObject::invokeMethod(camera, "loadCfg");
 }
 
@@ -183,7 +185,9 @@ void CameraWidget::preview(EyeData data)
     if (!shouldUpdate(data.timestamp))
         return;
     if (!isDataRecent(data.timestamp))
-        return;
+		return;
+
+	updateWidgetSize(data.input.cols, data.input.rows);
 
     QImage scaled = previewImage(data.input);
 
@@ -212,13 +216,15 @@ void CameraWidget::preview(const DataTuple &data)
     if (!isDataRecent(data.field.timestamp))
         return;
 
-    Mat input = data.field.input;
+	Mat input = data.field.input;
+
+	updateWidgetSize(input.cols, input.rows);
 
     if (data.showGazeEstimationVisualization) {
         if (data.gazeEstimationVisualization.empty())
             return;
         input = data.gazeEstimationVisualization;
-    }
+	}
 
     QImage scaled = previewImage(input);
 
@@ -236,29 +242,34 @@ void CameraWidget::preview(const DataTuple &data)
 
 void CameraWidget::updateFrameRate(Timestamp t)
 {
-    dt.push_back(t-lastTimestamp);
-    lastTimestamp = t;
-
-    if (dt.size() < 30) {
-        ui->statusbar->showMessage(
-            QString("%1 @ N/A FPS").arg(
-                camera->currentCameraInfo.description()
-                )
-            );
-        return;
-    }
-    dt.pop_front();
-
-    int acc = 0;
-    for (std::list<int>::iterator it=dt.begin(); it != dt.end(); ++it)
-        acc += *it;
-    double fps = 1000 / (double(acc)/dt.size());
-
-    ui->statusbar->showMessage(
-        QString("%1 @ %2 FPS").arg(
-            camera->currentCameraInfo.description()).arg(
-            fps, 0, 'f', 2)
-        );
+	// Simpler version since the status bar update showed some overhead
+	// during tests with OpenGL
+	if (tq.size() == 0) {
+		ui->statusbar->showMessage(
+			QString("%1 @ N/A FPS").arg(
+					camera->currentCameraInfo.description()
+				)
+			);
+		tq.push_back(t);
+		lastFrameRateUpdate = t;
+		return;
+	}
+
+	tq.push_back(t);
+	if (tq.size() > 30) {
+		tq.pop_front();
+		if (tq.back() - lastFrameRateUpdate > 100) { // update every 100 ms
+			double fps = (tq.size()-1) / (1.0e-3*( tq.back() - tq.front() ) );
+			lastFrameRateUpdate = tq.back();
+			ui->statusbar->showMessage(
+				QString("%1 @ %2 FPS").arg(
+					camera->currentCameraInfo.description()).arg(
+						fps, 0, 'f', 2
+					)
+				);
+		}
+	}
+
 }
 
 void CameraWidget::noCamera(QString msg)
@@ -349,7 +360,7 @@ void CameraWidget::mouseReleaseEvent(QMouseEvent *event)
             sROI = QPointF();
             eROI = QPointF();
         }
-        emit newROI(sROI, eROI);
+		setROI(sROI, eROI);
     }
 }
 
@@ -370,8 +381,18 @@ bool CameraWidget::shouldUpdate(Timestamp t)
     if (!this->isVisible())
         return false;
 
-    // TODO: Right now, we don't update every frame to save resources.
-    // Make this parametrizable or move to faster drawing methods
+	/* TODO: Right now, we don't update every frame to save resources.
+	 * Make this parametrizable or move to faster drawing methods.
+	 *
+	 * Notes: I've tried some initial tests to move to OpenGL, which were
+	 * unsuccessful. This includes using QPainter with QOpenGLWidgets, native
+	 * OpenGL directives (using QUADs, TRIANGLES, and an FBO).
+	 * In all cases I saw a similar behavior: scaling/drawing is faster, but I
+	 * get CPU hogs from other places -- e.g., additional calls to nvogl,
+	 * NtGdiDdDDIEscape (whatever that is!), and calls to wglMakeCurrent
+	 * (which seem to be rather expensive!).
+	 * If anyone who actually knows OpenGL wanna try, be my guest :-)
+	 */
     if (t - lastUpdate > updateIntervalMs) {
         lastUpdate = t;
         return true;
@@ -394,23 +415,38 @@ bool CameraWidget::isDataRecent(Timestamp t)
 
 QImage CameraWidget::previewImage(const cv::Mat &frame)
 {
-    Mat rgb;
     Size previewSize(ui->viewFinder->width(), ui->viewFinder->height());
 
     switch (frame.channels()) {
         case 1:
-            cvtColor(frame, rgb, CV_GRAY2RGB);
-            cv::resize(rgb, rgb, previewSize, CV_INTER_LINEAR);
-            break;
+			cvtColor(frame, rgb, CV_GRAY2RGB);
+			cv::resize(rgb, resized, previewSize, cv::INTER_NEAREST);
+			break;
         case 3:
-            cvtColor(frame, rgb, CV_BGR2RGB);
-            cv::resize(rgb, rgb, previewSize, CV_INTER_LINEAR);
+			cvtColor(frame, rgb, CV_BGR2RGB);
+			cv::resize(rgb, resized, previewSize, cv::INTER_NEAREST);
             break;
         default:
-            rgb = cv::Mat::zeros(previewSize, CV_8UC3);
+			resized = cv::Mat::zeros(previewSize, CV_8UC3);
             break;
-    }
-    QImage scaled = QImage(rgb.data, rgb.cols, rgb.rows, (int) rgb.step, QImage::Format_RGB888).copy();
+	}
+
+	/* TODO: according to http://doc.qt.io/qt-5/qpainter.html
+	 *
+	 * "Raster - This backend implements all rendering in pure software and is
+	 * always used to render into QImages. For optimal performance only use the
+	 * format types QImage::Format_ARGB32_Premultiplied, QImage::Format_RGB32 or
+	 * QImage::Format_RGB16. Any other format, including QImage::Format_ARGB32,
+	 * has significantly worse performance. This engine is used by default for
+	 * QWidget and QPixmap."
+	 *
+	 * However, I've found that resizing with OpenCV using only three bytes
+	 * results in faster and less cpu-intensive code.
+	 * This is relative to ARGB32_Premultiplied (which should be the fastest
+	 * option with QImages, according to http://doc.qt.io/qt-5/qimage.html
+	 *
+	 */
+	QImage scaled = QImage(resized.data, resized.cols, resized.rows, (int) resized.step, QImage::Format_RGB888);
 
     rw = scaled.width() / (float) frame.cols;
     rh = scaled.height() / (float) frame.rows;
@@ -510,3 +546,17 @@ void CameraWidget::drawGaze(const FieldData &field, QPainter &painter)
     painter.resetTransform();
 }
 
+void CameraWidget::updateWidgetSize(const int &width, const int &height)
+{
+	// Logic to limit the size of the camera widgets
+	QSize newFrameSize = { width, height };
+	if (frameSize == newFrameSize)
+		return;
+
+	frameSize = newFrameSize;
+	QSize maxSize = { 960 , 540 };
+	if (frameSize.width() < maxSize.width() && frameSize.height() < maxSize.height() )
+		this->setMaximumSize( frameSize );
+	else
+		this->setMaximumSize( maxSize );
+}

EyeRecToo/src/CameraWidget.h

@@ -35,7 +35,7 @@ class CameraWidget : public QMainWindow, InputWidget
 
 public:
     explicit CameraWidget(QString id, ImageProcessor::Type type, QWidget *parent = 0);
-    ~CameraWidget();
+	~CameraWidget();
 
 signals:
     void setCamera(QCameraInfo cameraInfo);
@@ -84,14 +84,15 @@ private:
     QActionGroup *optionsGroup;
     QAction *optionAction;
 
-    std::list<int> dt;
-    Timestamp lastTimestamp;
+	std::deque<int> tq;
+	Timestamp lastFrameRateUpdate;
     void updateFrameRate(Timestamp t);
 
     QPointF sROI, eROI;
-    bool settingROI;
+	bool settingROI;
+	void setROI(const QPointF &s, const QPointF &e) { sROI=s; eROI=e; emit newROI(sROI, eROI); }
 
-    Timestamp lastUpdate;
+	Timestamp lastUpdate;
     Timestamp updateIntervalMs;
     Timestamp maxAgeMs;
     bool shouldUpdate(Timestamp t);
@@ -102,7 +103,11 @@ private:
 	CameraCalibration *cameraCalibration;
 	bool cameraCalibrationSampleRequested;
 
-    // Drawing functions
+	QSize frameSize = { 0, 0 };
+	void updateWidgetSize( const int &width, const int &height);
+
+	// Drawing functions
+	cv::Mat rgb, resized;
     double rw, rh;
     double refPx;
     QFont font;

EyeRecToo/src/CameraWidget.ui

@@ -12,8 +12,8 @@
   </property>
   <property name="maximumSize">
    <size>
-    <width>854</width>
-    <height>480</height>
+    <width>16777215</width>
+    <height>16777215</height>
    </size>
   </property>
   <property name="windowTitle">

EyeRecToo/src/DataRecorder.cpp

@@ -139,5 +139,7 @@ void DataRecorder::storeData(T &data)
     if (videoWriter->isOpened())
         *videoWriter << data.input;
     if (dataStream->status() == QTextStream::Ok)
-        *dataStream << data.toQString() << gDataNewline;
+		*dataStream << data.toQString() << gDataNewline;
+
+	// TODO: add recording timestamp?
 }

EyeRecToo/src/EyeImageProcessor.cpp

@@ -49,8 +49,6 @@ EyeImageProcessor::~EyeImageProcessor()
 
 void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
 {
-    Mat prevInput = data.input;
-
     // TODO: parametrize frame drop due to lack of processing power
     if ( gPerformanceMonitor.shouldDrop(pmIdx, gTimer.elapsed() - timestamp, 50) )
         return;
@@ -59,13 +57,14 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
 
     data.timestamp = timestamp;
 
-    // Always force the creation of a new matrix for the input since the old one might still be alive from further down the pipeline
-    if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
-        data.input = Mat::zeros(cfg.inputSize, frame.type() );
-        resize(frame, data.input, cfg.inputSize);
-    }
-    else
-        data.input = frame.clone();
+	Q_ASSERT_X(frame.data != data.input.data, "Eye Image Processing", "Previous and current input image matches!");
+	if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
+		data.input = Mat(cfg.inputSize, frame.type() );
+		resize(frame, data.input, cfg.inputSize);
+	}
+	else {
+		data.input = frame;
+	}
 
     if (cfg.flip != CV_FLIP_NONE)
         flip(data.input, data.input, cfg.flip);
@@ -111,9 +110,8 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
         }
     }
 
-    data.processingTimestamp = gTimer.elapsed();
+	data.processingTimestamp = gTimer.elapsed() - data.timestamp;
 
-    Q_ASSERT_X(prevInput.data != data.input.data, "Eye Image Processing", "Previous and current input image matches!");
     emit newData(data);
 }
 

EyeRecToo/src/FieldImageProcessor.cpp

@@ -41,8 +41,6 @@ FieldImageProcessor::~FieldImageProcessor()
 
 void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
 {
-    Mat prevInput = data.input;
-
     // TODO: parametrize frame drop due to lack of processing power
     if ( gPerformanceMonitor.shouldDrop(pmIdx, gTimer.elapsed() - timestamp, 100) )
         return;
@@ -51,13 +49,14 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
 
     data.timestamp = timestamp;
 
-    // Always force the creation of a new matrix for the input since the old one might still be alive from further down the pipeline
     if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
         data.input = Mat(cfg.inputSize, frame.type() );
         resize(frame, data.input, cfg.inputSize);
     }
-    else
-        data.input = frame.clone();
+	else {
+		Q_ASSERT_X(frame.data != data.input.data, "Field Image Processing", "Previous and current input image matches!");
+		data.input = frame;
+	}
 
     if (cfg.flip != CV_FLIP_NONE)
         flip(data.input, data.input, cfg.flip);
@@ -74,21 +73,26 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
     // Marker detection and pose estimation
     vector<int> ids;
     vector<vector<Point2f> > corners;
-    if (cfg.processingDownscalingFactor > 1) {
-        Mat downscaled;
+	Mat downscaled;
+	if (cfg.processingDownscalingFactor > 1) {
         resize(data.input, downscaled, Size(),
                1/cfg.processingDownscalingFactor,
                1/cfg.processingDownscalingFactor,
                INTER_AREA);
-        if (cfg.markerDetectionMethod == "aruco") {
-            detectMarkers(downscaled, dict, corners, ids, detectorParameters);
-            for (unsigned int i=0; i<ids.size(); i++)
-                for (unsigned int j=0; j<corners[i].size(); j++)
-                    corners[i][j] = cfg.processingDownscalingFactor*corners[i][j];
-        }
-    } else
-        if (cfg.markerDetectionMethod == "aruco")
-            detectMarkers(data.input, dict, corners, ids, detectorParameters);
+	} else {
+		downscaled = data.input;
+	}
+
+	if (cfg.markerDetectionMethod == "aruco") {
+
+		detectMarkers(downscaled, dict, corners, ids, detectorParameters);
+
+		if (cfg.processingDownscalingFactor > 1) { // Upscale if necessary
+			for (unsigned int i=0; i<ids.size(); i++)
+				for (unsigned int j=0; j<corners[i].size(); j++)
+					corners[i][j] = cfg.processingDownscalingFactor*corners[i][j];
+		}
+	}
 
     // Filling the marker data
     data.collectionMarker = Marker();
@@ -131,9 +135,8 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
     }
 
     data.validGazeEstimate = false;
-    data.processingTimestamp = gTimer.elapsed();
+	data.processingTimestamp = gTimer.elapsed() - data.timestamp;
 
-    Q_ASSERT_X(prevInput.data != data.input.data, "Field Image Processing", "Previous and current input image matches!");
     emit newData(data);
 }
 

EyeRecToo/src/FrameGrabber.cpp

@@ -160,7 +160,7 @@ bool FrameGrabber::jpeg2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
         return false;
     }
 
-    cvFrame = Mat::zeros(height, width, code);
+	cvFrame = Mat(height, width, code);
     int decode = code == CV_8UC3 ? TJPF_BGR : TJPF_GRAY;
     res = tjDecodeYUV(tjh, yuvBuffer, 4, subsamp, cvFrame.data, width, 0, height, decode, 0);
     if (res < 0)
@@ -193,7 +193,7 @@ bool FrameGrabber::rgb32_2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
 bool FrameGrabber::yuyv_2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
 {
     // TODO: can we optimize this?
-    cvFrame = Mat::zeros(abs(in.height()), abs(in.width()), CV_8UC3);
+	cvFrame = Mat(abs(in.height()), abs(in.width()), CV_8UC3);
     const unsigned char *pyuv = in.bits();
     unsigned char *pbgr = cvFrame.data;
     for(int i = 0; i < in.mappedBytes(); i += 4) {

EyeRecToo/src/main.cpp

@@ -43,7 +43,12 @@ int main(int argc, char *argv[])
 
     makeFolders();
 
-    QApplication a(argc, argv);
+	/*
+	QCoreApplication::setAttribute(Qt::AA_UseDesktopOpenGL, false);
+	QCoreApplication::setAttribute(Qt::AA_UseOpenGLES, false);
+	QCoreApplication::setAttribute(Qt::AA_UseSoftwareOpenGL, true);
+	*/
+	QApplication a(argc, argv);
     a.setStyle(QStyleFactory::create("Fusion"));
 
 //#define DARK_THEME