Commit 5d6cc81c authored by Thiago Santini's avatar Thiago Santini

Adds performance improvements

Code further profiled, and removed a couple unnecessary operations.

Added information about possible move to OpenGL for the camera widgets.

Fixed bug for eye processor pre-processing downscaling.
parent 9ded3ef2
......@@ -8,7 +8,6 @@ CameraWidget::CameraWidget(QString id, ImageProcessor::Type type, QWidget *paren
QMainWindow(parent),
id(id),
type(type),
lastTimestamp(0),
sROI(QPoint(0,0)),
eROI(QPoint(0,0)),
settingROI(false),
......@@ -122,7 +121,10 @@ CameraWidget::CameraWidget(QString id, ImageProcessor::Type type, QWidget *paren
connect(this, SIGNAL(newROI(QPointF,QPointF)),
imageProcessor, SIGNAL(newROI(QPointF,QPointF)) );
font.setStyleHint(QFont::Monospace);
// Initial roi
setROI( QPointF(0.1, 0.1), QPointF(0.9, 0.9) );
font.setStyleHint(QFont::Monospace);
QMetaObject::invokeMethod(camera, "loadCfg");
}
......@@ -183,7 +185,9 @@ void CameraWidget::preview(EyeData data)
if (!shouldUpdate(data.timestamp))
return;
if (!isDataRecent(data.timestamp))
return;
return;
updateWidgetSize(data.input.cols, data.input.rows);
QImage scaled = previewImage(data.input);
......@@ -212,13 +216,15 @@ void CameraWidget::preview(const DataTuple &data)
if (!isDataRecent(data.field.timestamp))
return;
Mat input = data.field.input;
Mat input = data.field.input;
updateWidgetSize(input.cols, input.rows);
if (data.showGazeEstimationVisualization) {
if (data.gazeEstimationVisualization.empty())
return;
input = data.gazeEstimationVisualization;
}
}
QImage scaled = previewImage(input);
......@@ -236,29 +242,34 @@ void CameraWidget::preview(const DataTuple &data)
void CameraWidget::updateFrameRate(Timestamp t)
{
dt.push_back(t-lastTimestamp);
lastTimestamp = t;
if (dt.size() < 30) {
ui->statusbar->showMessage(
QString("%1 @ N/A FPS").arg(
camera->currentCameraInfo.description()
)
);
return;
}
dt.pop_front();
int acc = 0;
for (std::list<int>::iterator it=dt.begin(); it != dt.end(); ++it)
acc += *it;
double fps = 1000 / (double(acc)/dt.size());
ui->statusbar->showMessage(
QString("%1 @ %2 FPS").arg(
camera->currentCameraInfo.description()).arg(
fps, 0, 'f', 2)
);
// Simpler version since the status bar update showed some overhead
// during tests with OpenGL
if (tq.size() == 0) {
ui->statusbar->showMessage(
QString("%1 @ N/A FPS").arg(
camera->currentCameraInfo.description()
)
);
tq.push_back(t);
lastFrameRateUpdate = t;
return;
}
tq.push_back(t);
if (tq.size() > 30) {
tq.pop_front();
if (tq.back() - lastFrameRateUpdate > 100) { // update every 100 ms
double fps = (tq.size()-1) / (1.0e-3*( tq.back() - tq.front() ) );
lastFrameRateUpdate = tq.back();
ui->statusbar->showMessage(
QString("%1 @ %2 FPS").arg(
camera->currentCameraInfo.description()).arg(
fps, 0, 'f', 2
)
);
}
}
}
void CameraWidget::noCamera(QString msg)
......@@ -349,7 +360,7 @@ void CameraWidget::mouseReleaseEvent(QMouseEvent *event)
sROI = QPointF();
eROI = QPointF();
}
emit newROI(sROI, eROI);
setROI(sROI, eROI);
}
}
......@@ -370,8 +381,18 @@ bool CameraWidget::shouldUpdate(Timestamp t)
if (!this->isVisible())
return false;
// TODO: Right now, we don't update every frame to save resources.
// Make this parametrizable or move to faster drawing methods
/* TODO: Right now, we don't update every frame to save resources.
* Make this parametrizable or move to faster drawing methods.
*
* Notes: I've tried some initial tests to move to OpenGL, which were
* unsuccessful. This includes using QPainter with QOpenGLWidgets, native
* OpenGL directives (using QUADs, TRIANGLES, and an FBO).
* In all cases I saw a similar behavior: scaling/drawing is faster, but I
* get CPU hogs from other places -- e.g., additional calls to nvogl,
* NtGdiDdDDIEscape (whatever that is!), and calls to wglMakeCurrent
* (which seem to be rather expensive!).
* If anyone who actually knows OpenGL wanna try, be my guest :-)
*/
if (t - lastUpdate > updateIntervalMs) {
lastUpdate = t;
return true;
......@@ -394,23 +415,38 @@ bool CameraWidget::isDataRecent(Timestamp t)
QImage CameraWidget::previewImage(const cv::Mat &frame)
{
Mat rgb;
Size previewSize(ui->viewFinder->width(), ui->viewFinder->height());
switch (frame.channels()) {
case 1:
cvtColor(frame, rgb, CV_GRAY2RGB);
cv::resize(rgb, rgb, previewSize, CV_INTER_LINEAR);
break;
cvtColor(frame, rgb, CV_GRAY2RGB);
cv::resize(rgb, resized, previewSize, cv::INTER_NEAREST);
break;
case 3:
cvtColor(frame, rgb, CV_BGR2RGB);
cv::resize(rgb, rgb, previewSize, CV_INTER_LINEAR);
cvtColor(frame, rgb, CV_BGR2RGB);
cv::resize(rgb, resized, previewSize, cv::INTER_NEAREST);
break;
default:
rgb = cv::Mat::zeros(previewSize, CV_8UC3);
resized = cv::Mat::zeros(previewSize, CV_8UC3);
break;
}
QImage scaled = QImage(rgb.data, rgb.cols, rgb.rows, (int) rgb.step, QImage::Format_RGB888).copy();
}
/* TODO: according to http://doc.qt.io/qt-5/qpainter.html
*
* "Raster - This backend implements all rendering in pure software and is
* always used to render into QImages. For optimal performance only use the
* format types QImage::Format_ARGB32_Premultiplied, QImage::Format_RGB32 or
* QImage::Format_RGB16. Any other format, including QImage::Format_ARGB32,
* has significantly worse performance. This engine is used by default for
* QWidget and QPixmap."
*
* However, I've found that resizing with OpenCV using only three bytes
* results in faster and less cpu-intensive code.
* This is relative to ARGB32_Premultiplied (which should be the fastest
* option with QImages, according to http://doc.qt.io/qt-5/qimage.html
*
*/
QImage scaled = QImage(resized.data, resized.cols, resized.rows, (int) resized.step, QImage::Format_RGB888);
rw = scaled.width() / (float) frame.cols;
rh = scaled.height() / (float) frame.rows;
......@@ -510,3 +546,17 @@ void CameraWidget::drawGaze(const FieldData &field, QPainter &painter)
painter.resetTransform();
}
void CameraWidget::updateWidgetSize(const int &width, const int &height)
{
// Logic to limit the size of the camera widgets
QSize newFrameSize = { width, height };
if (frameSize == newFrameSize)
return;
frameSize = newFrameSize;
QSize maxSize = { 960 , 540 };
if (frameSize.width() < maxSize.width() && frameSize.height() < maxSize.height() )
this->setMaximumSize( frameSize );
else
this->setMaximumSize( maxSize );
}
......@@ -35,7 +35,7 @@ class CameraWidget : public QMainWindow, InputWidget
public:
explicit CameraWidget(QString id, ImageProcessor::Type type, QWidget *parent = 0);
~CameraWidget();
~CameraWidget();
signals:
void setCamera(QCameraInfo cameraInfo);
......@@ -84,14 +84,15 @@ private:
QActionGroup *optionsGroup;
QAction *optionAction;
std::list<int> dt;
Timestamp lastTimestamp;
std::deque<int> tq;
Timestamp lastFrameRateUpdate;
void updateFrameRate(Timestamp t);
QPointF sROI, eROI;
bool settingROI;
bool settingROI;
void setROI(const QPointF &s, const QPointF &e) { sROI=s; eROI=e; emit newROI(sROI, eROI); }
Timestamp lastUpdate;
Timestamp lastUpdate;
Timestamp updateIntervalMs;
Timestamp maxAgeMs;
bool shouldUpdate(Timestamp t);
......@@ -102,7 +103,11 @@ private:
CameraCalibration *cameraCalibration;
bool cameraCalibrationSampleRequested;
// Drawing functions
QSize frameSize = { 0, 0 };
void updateWidgetSize( const int &width, const int &height);
// Drawing functions
cv::Mat rgb, resized;
double rw, rh;
double refPx;
QFont font;
......
......@@ -12,8 +12,8 @@
</property>
<property name="maximumSize">
<size>
<width>854</width>
<height>480</height>
<width>16777215</width>
<height>16777215</height>
</size>
</property>
<property name="windowTitle">
......
......@@ -139,5 +139,7 @@ void DataRecorder::storeData(T &data)
if (videoWriter->isOpened())
*videoWriter << data.input;
if (dataStream->status() == QTextStream::Ok)
*dataStream << data.toQString() << gDataNewline;
*dataStream << data.toQString() << gDataNewline;
// TODO: add recording timestamp?
}
......@@ -49,8 +49,6 @@ EyeImageProcessor::~EyeImageProcessor()
void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
{
Mat prevInput = data.input;
// TODO: parametrize frame drop due to lack of processing power
if ( gPerformanceMonitor.shouldDrop(pmIdx, gTimer.elapsed() - timestamp, 50) )
return;
......@@ -59,13 +57,14 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
data.timestamp = timestamp;
// Always force the creation of a new matrix for the input since the old one might still be alive from further down the pipeline
if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
data.input = Mat::zeros(cfg.inputSize, frame.type() );
resize(frame, data.input, cfg.inputSize);
}
else
data.input = frame.clone();
Q_ASSERT_X(frame.data != data.input.data, "Eye Image Processing", "Previous and current input image matches!");
if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
data.input = Mat(cfg.inputSize, frame.type() );
resize(frame, data.input, cfg.inputSize);
}
else {
data.input = frame;
}
if (cfg.flip != CV_FLIP_NONE)
flip(data.input, data.input, cfg.flip);
......@@ -111,9 +110,8 @@ void EyeImageProcessor::process(Timestamp timestamp, const Mat &frame)
}
}
data.processingTimestamp = gTimer.elapsed();
data.processingTimestamp = gTimer.elapsed() - data.timestamp;
Q_ASSERT_X(prevInput.data != data.input.data, "Eye Image Processing", "Previous and current input image matches!");
emit newData(data);
}
......
......@@ -41,8 +41,6 @@ FieldImageProcessor::~FieldImageProcessor()
void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
{
Mat prevInput = data.input;
// TODO: parametrize frame drop due to lack of processing power
if ( gPerformanceMonitor.shouldDrop(pmIdx, gTimer.elapsed() - timestamp, 100) )
return;
......@@ -51,13 +49,14 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
data.timestamp = timestamp;
// Always force the creation of a new matrix for the input since the old one might still be alive from further down the pipeline
if (cfg.inputSize.width > 0 && cfg.inputSize.height > 0) {
data.input = Mat(cfg.inputSize, frame.type() );
resize(frame, data.input, cfg.inputSize);
}
else
data.input = frame.clone();
else {
Q_ASSERT_X(frame.data != data.input.data, "Field Image Processing", "Previous and current input image matches!");
data.input = frame;
}
if (cfg.flip != CV_FLIP_NONE)
flip(data.input, data.input, cfg.flip);
......@@ -74,21 +73,26 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
// Marker detection and pose estimation
vector<int> ids;
vector<vector<Point2f> > corners;
if (cfg.processingDownscalingFactor > 1) {
Mat downscaled;
Mat downscaled;
if (cfg.processingDownscalingFactor > 1) {
resize(data.input, downscaled, Size(),
1/cfg.processingDownscalingFactor,
1/cfg.processingDownscalingFactor,
INTER_AREA);
if (cfg.markerDetectionMethod == "aruco") {
detectMarkers(downscaled, dict, corners, ids, detectorParameters);
for (unsigned int i=0; i<ids.size(); i++)
for (unsigned int j=0; j<corners[i].size(); j++)
corners[i][j] = cfg.processingDownscalingFactor*corners[i][j];
}
} else
if (cfg.markerDetectionMethod == "aruco")
detectMarkers(data.input, dict, corners, ids, detectorParameters);
} else {
downscaled = data.input;
}
if (cfg.markerDetectionMethod == "aruco") {
detectMarkers(downscaled, dict, corners, ids, detectorParameters);
if (cfg.processingDownscalingFactor > 1) { // Upscale if necessary
for (unsigned int i=0; i<ids.size(); i++)
for (unsigned int j=0; j<corners[i].size(); j++)
corners[i][j] = cfg.processingDownscalingFactor*corners[i][j];
}
}
// Filling the marker data
data.collectionMarker = Marker();
......@@ -131,9 +135,8 @@ void FieldImageProcessor::process(Timestamp timestamp, const Mat &frame)
}
data.validGazeEstimate = false;
data.processingTimestamp = gTimer.elapsed();
data.processingTimestamp = gTimer.elapsed() - data.timestamp;
Q_ASSERT_X(prevInput.data != data.input.data, "Field Image Processing", "Previous and current input image matches!");
emit newData(data);
}
......
......@@ -160,7 +160,7 @@ bool FrameGrabber::jpeg2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
return false;
}
cvFrame = Mat::zeros(height, width, code);
cvFrame = Mat(height, width, code);
int decode = code == CV_8UC3 ? TJPF_BGR : TJPF_GRAY;
res = tjDecodeYUV(tjh, yuvBuffer, 4, subsamp, cvFrame.data, width, 0, height, decode, 0);
if (res < 0)
......@@ -193,7 +193,7 @@ bool FrameGrabber::rgb32_2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
bool FrameGrabber::yuyv_2bmp(const QVideoFrame &in, cv::Mat &cvFrame)
{
// TODO: can we optimize this?
cvFrame = Mat::zeros(abs(in.height()), abs(in.width()), CV_8UC3);
cvFrame = Mat(abs(in.height()), abs(in.width()), CV_8UC3);
const unsigned char *pyuv = in.bits();
unsigned char *pbgr = cvFrame.data;
for(int i = 0; i < in.mappedBytes(); i += 4) {
......
......@@ -43,7 +43,12 @@ int main(int argc, char *argv[])
makeFolders();
QApplication a(argc, argv);
/*
QCoreApplication::setAttribute(Qt::AA_UseDesktopOpenGL, false);
QCoreApplication::setAttribute(Qt::AA_UseOpenGLES, false);
QCoreApplication::setAttribute(Qt::AA_UseSoftwareOpenGL, true);
*/
QApplication a(argc, argv);
a.setStyle(QStyleFactory::create("Fusion"));
//#define DARK_THEME
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment