WIP: Dev openglaccelerated playback

YUViewLib/YUViewLib.pro

@@ -14,7 +14,8 @@ INCLUDEPATH += src/
 
 RESOURCES += \
     images/images.qrc \
-    docs/docs.qrc
+    docs/docs.qrc \
+    shaders/shaders.qrc
 
 contains(QT_ARCH, x86_32|i386) {
     warning("You are building for a 32 bit system. This is untested and not supported.")

YUViewLib/shaders/fragmentYUV2RGBshader.glsl

@@ -0,0 +1,69 @@
+#version 330 core
+
+// A Fragment Shader is the Shader stage that will process a Fragment generated by the
+// Rasterization into a set of colors and a single depth value.
+// The fragment shader is the OpenGL pipeline stage after a primitive is rasterized.
+// For each sample of the pixels covered by a primitive, a "fragment" is generated.
+// Each fragment has a Window Space position, a few other values, and it contains
+// all of the interpolated per-vertex output values from the last Vertex Processing stage.
+// The output of a fragment shader is a depth value, a possible stencil value
+// (unmodified by the fragment shader), and zero or more color values to be
+// potentially written to the buffers in the current framebuffers.
+// Fragment shaders take a single fragment as input and produce a single fragment as output.
+
+// Interpolated values from the vertex shaders
+in vec2 texture_coordinate;
+//in vec2 fragmentUVChroma;
+
+// Ouput data
+out vec4 color_0;
+//out vec3 color_1;
+
+// Values that stay constant for the whole mesh.
+uniform usampler2D textureSamplerY;
+uniform usampler2D textureSamplerU;
+uniform usampler2D textureSamplerV;
+
+uniform float sampleMaxVal;
+
+void main(){
+    // ITU-R BT 709 to RGB mapping matrix
+    mat3 mat709toRGB = mat3(1.164, 1.164, 1.164,  // definition looks transposed
+                            0.0, -0.213, 2.112,   // since openGL uses column-major order
+                            1.793, -0.533, 0.0);
+
+
+
+    // Output color = color of the texture at the specified UV
+    vec3 color709;
+    //color709.rgb = texture2D( textureSamplerYUV, fragmentUV ).rgb;
+    color709.r = float(texture( textureSamplerY, texture_coordinate ).r);
+    color709.g = float(texture( textureSamplerU, texture_coordinate ).r);
+    color709.b = float(texture( textureSamplerV, texture_coordinate ).r);
+
+    // normalize color values to the intervall [0, 1]
+    color709.r = color709.r / sampleMaxVal;
+    color709.g = color709.g / sampleMaxVal;
+    color709.b = color709.b / sampleMaxVal;
+
+// obsolete, but still missing atm.
+//    // make sure input has correct range, 8bit
+//    color709.r = clamp(color709.r, 16.0/255, 235.0/255); // Y
+//    color709.g = clamp(color709.g, 16.0/255, 240.0/255); // U
+//    color709.b = clamp(color709.b, 16.0/255, 240.0/255); // V
+//    // make sure input has correct range, 10bit
+//    color709.r = clamp(color709.r, 16.0/1023, 235.0/1023); // Y
+//    color709.g = clamp(color709.g, 16.0/1023, 240.0/1023); // U
+//    color709.b = clamp(color709.b, 16.0/1023, 240.0/1023); // V
+
+
+    // de/normalization
+    color709.r = color709.r - 0.0625;
+    color709.g = color709.g - 0.5;
+    color709.b = color709.b - 0.5;
+    // finally the conversion
+    color_0 = vec4(clamp(mat709toRGB * color709, 0.0, 1.0), 1.0f);
+    //color_1.r = 0.5;
+    //color_1.g = 0.7;
+    //color_1.b = 0.8;
+}

YUViewLib/shaders/shaders.qrc

@@ -0,0 +1,6 @@
+<RCC>
+    <qresource prefix="/">
+        <file>fragmentYUV2RGBshader.glsl</file>
+        <file>vertexshader.glsl</file>
+    </qresource>
+</RCC>

YUViewLib/shaders/vertexshader.glsl

@@ -0,0 +1,30 @@
+#version 330 core
+
+// The Vertex Shader is the programmable Shader stage in the rendering pipeline that
+// handles the processing of individual vertices. Vertex shaders are fed
+// Vertex Attribute data, as specified from a vertex array object by a drawing command.
+// A vertex shader receives a single vertex from the vertex stream and generates
+// a single vertex to the output vertex stream. There must be a 1:1 mapping from
+// input vertices to output vertices.
+
+// Input vertex data, different for all executions of this shader.
+layout(location = 0) in vec3 vertexPosition_modelspace;
+layout(location = 1) in vec2 vertexLuma;
+
+// Output data ; will be interpolated for each fragment.
+out vec2 texture_coordinate; // texture coordinates
+
+// Mmodel view projection (MVP) matrix
+// the vertices are setup to be aligned with openGL clip coordinates.
+// Thus no further coordinate transform will be necessary.
+// Further, textrue coordinates can be obtained by dropping the z-axis coordinate.
+// https://learnopengl.com/Getting-started/Coordinate-Systems
+// However we need to scale in case the aspect ratio of the window does not match the videos
+uniform mat4 mvp_matrix;
+
+
+void main(){        
+    gl_Position = mvp_matrix * vec4(vertexPosition_modelspace, 1.0f);
+    // for texture coordinates we simply discard the z axis, and flip the y axis
+    texture_coordinate = vec2(vertexLuma.x, 1.0 - vertexLuma.y);
+}

YUViewLib/src/playlistitem/playlistItem.h

@@ -40,6 +40,7 @@
 #include "common/YUViewDomElement.h"
 
 #include "ui_playlistItem.h"
+#include <video/yuvPixelFormat.h>
 
 class frameHandler;
 class statisticHandler;
@@ -230,6 +231,9 @@ class playlistItem : public QObject, public QTreeWidgetItem
   // This will trigger the tree widget to update it's contents.
   void signalItemChanged(bool redraw, recacheIndicator recache);
 
+  // for sending new data to openGL view
+  void signalNewFrame(const int frameWidth, const int frameHeight, const YUV_Internals::yuvPixelFormat newFormat,  const QByteArray &newPicture);
+
   // The item finished loading a frame into the double buffer. This is relevant if playback is paused and waiting
   // for the item to load the next frame into the double buffer. This will restart the timer. 
   void signalItemDoubleBufferLoaded();

YUViewLib/src/playlistitem/playlistItemCompressedVideo.cpp

@@ -982,7 +982,10 @@ void playlistItemCompressedVideo::loadFrame(int frameIdx, bool playing, bool loa
 
     isFrameLoading = false;
     if (emitSignals)
+    {
       emit signalItemChanged(true, RECACHE_NONE);
+      emit signalNewFrame(video->getFrameSize().width(), video->getFrameSize().height(), video->getFormat(), video->currentFrameRawData);
+    }
   }
 
   if (playing && (stateYUV == LoadingNeeded || stateYUV == LoadingNeededDoubleBuffer))
@@ -996,7 +999,10 @@ void playlistItemCompressedVideo::loadFrame(int frameIdx, bool playing, bool loa
       video->loadFrame(nextFrameIdx, true);
       isFrameLoadingDoubleBuffer = false;
       if (emitSignals)
+      {
         emit signalItemDoubleBufferLoaded();
+        emit signalNewFrame(video->getFrameSize().width(), video->getFrameSize().height(), video->getFormat(), video->currentFrameRawData);
+      }
     }
   }
 }

YUViewLib/src/playlistitem/playlistItemWithVideo.cpp

@@ -86,7 +86,10 @@ void playlistItemWithVideo::loadFrame(int frameIdx, bool playing, bool loadRawDa
     video->loadFrame(frameIdxInternal);
     isFrameLoading = false;
     if (emitSignals)
+    {
       emit signalItemChanged(true, RECACHE_NONE);
+      emit signalNewFrame(video->getFrameSize().width(), video->getFrameSize().height(), video->getFormat(), video->currentFrameRawData);
+    }
   }
 
   if (playing && (state == LoadingNeeded || state == LoadingNeededDoubleBuffer))
@@ -100,7 +103,10 @@ void playlistItemWithVideo::loadFrame(int frameIdx, bool playing, bool loadRawDa
       video->loadFrame(nextFrameIdx, true);
       isFrameLoadingDoubleBuffer = false;
       if (emitSignals)
+      {
         emit signalItemDoubleBufferLoaded();
+        emit signalNewFrame(video->getFrameSize().width(), video->getFrameSize().height(), video->getFormat(), video->currentFrameRawData);
+      }
     }
   }
 }

YUViewLib/src/ui/mainwindow.cpp

@@ -51,6 +51,8 @@ MainWindow::MainWindow(bool useAlternativeSources, QWidget *parent) : QMainWindo
   Q_INIT_RESOURCE(images);
   Q_INIT_RESOURCE(docs);
 
+  qRegisterMetaType<YUV_Internals::yuvPixelFormat>();
+
   SettingsDialog::initializeDefaults();
 
   QSettings settings;
@@ -73,12 +75,19 @@ MainWindow::MainWindow(bool useAlternativeSources, QWidget *parent) : QMainWindo
   separateViewWindow.setWindowTitle("Separate View");
   separateViewWindow.setGeometry(0, 0, 300, 600);
 
+  // Initialize the OpenGL window
+  openGLWindow.setWindowTitle("OpenGL View");
+  openGLWindow.setGeometry(0, 0, 300, 600);
+
   connect(ui.displaySplitView, &splitViewWidget::signalToggleFullScreen, this, &MainWindow::toggleFullscreen);
 
   // Setup primary/separate splitView
   ui.displaySplitView->addSlaveView(&separateViewWindow.splitView);
   connect(ui.displaySplitView, &splitViewWidget::signalShowSeparateWindow, &separateViewWindow, &QWidget::setVisible);
 
+  // Setup openGL view
+  connect(ui.displaySplitView, &splitViewWidget::signalShowOpenGLWindow, this, &MainWindow::showOpenGLWindow);
+
   // Connect the playlistWidget signals to some slots
   auto const fileInfoAdapter = [this]{
     auto items = ui.playlistTreeWidget->getSelectedItems();
@@ -97,6 +106,7 @@ MainWindow::MainWindow(bool useAlternativeSources, QWidget *parent) : QMainWindo
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::selectionRangeChanged, ui.bitstreamAnalysis, &BitstreamAnalysisWidget::currentSelectedItemsChanged);
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::selectionRangeChanged, this, &MainWindow::currentSelectedItemsChanged);
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::selectedItemChanged, ui.playbackController, &PlaybackController::selectionPropertiesChanged);
+
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::itemAboutToBeDeleted, ui.propertiesWidget, &PropertiesWidget::itemAboutToBeDeleted);
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::openFileDialog, this, &MainWindow::showFileOpenDialog);
   connect(ui.playlistTreeWidget, &PlaylistTreeWidget::selectedItemDoubleBufferLoad, ui.playbackController, &PlaybackController::currentSelectedItemsDoubleBufferLoad);
@@ -110,6 +120,7 @@ MainWindow::MainWindow(bool useAlternativeSources, QWidget *parent) : QMainWindo
   createMenusAndActions();
 
   ui.playbackController->setSplitViews(ui.displaySplitView, &separateViewWindow.splitView);
+  ui.playbackController->setOpenGLView( &openGLWindow.openGLView );
   ui.playbackController->setPlaylist(ui.playlistTreeWidget);
   ui.displaySplitView->setPlaybackController(ui.playbackController);
   ui.displaySplitView->setPlaylistTreeWidget(ui.playlistTreeWidget);
@@ -350,6 +361,9 @@ void MainWindow::closeEvent(QCloseEvent *event)
 
   if (!separateViewWindow.isHidden())
     separateViewWindow.close();
+
+  if (!openGLWindow.isHidden())
+    openGLWindow.close();
 }
 
 void MainWindow::openRecentFile()
@@ -588,6 +602,13 @@ void MainWindow::showAboutHelp(bool showAbout)
   about->show();
 }
 
+void MainWindow::showOpenGLWindow()
+{
+    openGLWindow.setVisible(true);
+    connect(ui.playlistTreeWidget, &PlaylistTreeWidget::signalNewFrame, &openGLWindow.openGLView, &OpenGLViewWidget::updateFrame );
+}
+
+
 void MainWindow::showSettingsWindow()
 {
   SettingsDialog dialog;

YUViewLib/src/ui/mainwindow.h

@@ -37,6 +37,7 @@
 #include <QSettings>
 
 #include "ui/separateWindow.h"
+#include "ui/openGLWindow.h"
 #include "handler/updateHandler.h"
 #include "video/videoCache.h"
 
@@ -81,6 +82,7 @@ private slots:
   void showFileOpenDialog();
   void resetWindowLayout();
   void closeAndClearSettings();
+  void showOpenGLWindow();
 
   void onMenuResetView(bool checked) { const auto v = this->getCurrentActiveView(); if (v) v->resetView(checked); }
   void onMenuZoomToFit(bool checked) { const auto v = this->getCurrentActiveView(); if (v) v->zoomToFit(checked); }
@@ -132,6 +134,7 @@ private slots:
   QScopedPointer<updateHandler> updater;
   ViewStateHandler stateHandler;
   SeparateWindow separateViewWindow;
+  OpenGLWindow openGLWindow;
   bool showNormalMaximized; // When going to full screen: Was this windows maximized?  
   bool panelsVisible[5] {false};  // Which panels are visible when going to full-screen mode?
 };

YUViewLib/src/ui/openGLWindow.cpp

@@ -0,0 +1,109 @@
+/*  This file is part of YUView - The YUV player with advanced analytics toolset
+*   <https://github.com/IENT/YUView>
+*   Copyright (C) 2015  Institut für Nachrichtentechnik, RWTH Aachen University, GERMANY
+*
+*   This program is free software; you can redistribute it and/or modify
+*   it under the terms of the GNU General Public License as published by
+*   the Free Software Foundation; either version 3 of the License, or
+*   (at your option) any later version.
+*
+*   In addition, as a special exception, the copyright holders give
+*   permission to link the code of portions of this program with the
+*   OpenSSL library under certain conditions as described in each
+*   individual source file, and distribute linked combinations including
+*   the two.
+*   
+*   You must obey the GNU General Public License in all respects for all
+*   of the code used other than OpenSSL. If you modify file(s) with this
+*   exception, you may extend this exception to your version of the
+*   file(s), but you are not obligated to do so. If you do not wish to do
+*   so, delete this exception statement from your version. If you delete
+*   this exception statement from all source files in the program, then
+*   also delete it here.
+*
+*   This program is distributed in the hope that it will be useful,
+*   but WITHOUT ANY WARRANTY; without even the implied warranty of
+*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+*   GNU General Public License for more details.
+*
+*   You should have received a copy of the GNU General Public License
+*   along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "openGLWindow.h"
+
+#include <QSettings>
+
+OpenGLWindow::OpenGLWindow() :
+  openGLView(this)
+{
+
+
+    // https://doc.qt.io/qt-5/qopenglwidget.html
+    //As described above, it is simpler and more robust to set the requested format globally so
+    //  that it applies to all windows and contexts during the lifetime of the application.
+      QSurfaceFormat format;
+      // asks for a OpenGL 3.2 debug context using the Core profile
+      format.setMajorVersion(3);
+      format.setMinorVersion(3);
+      format.setDepthBufferSize(24);
+      format.setStencilBufferSize(8);
+      // Setting an interval value of 0 will turn the vertical refresh syncing off, any value
+      // higher than 0 will turn the vertical syncing on. Setting interval to a higher value,
+      // for example 10, results in having 10 vertical retraces between every buffer swap.
+      format.setSwapInterval(0); // no effect in windows
+      format.setProfile(QSurfaceFormat::CoreProfile);
+      format.setOption(QSurfaceFormat::DebugContext);
+      QSurfaceFormat::setDefaultFormat(format);
+
+
+    setCentralWidget(&openGLView);
+//  splitView.setAttribute(Qt::WA_AcceptTouchEvents);
+
+
+
+//  connect(&splitView, &splitViewWidget::signalToggleFullScreen, this, &OpenGLWindow::toggleFullscreen);
+//  connect(&splitView, &splitViewWidget::signalShowOpenGLWindow, this, &OpenGLWindow::splitViewShowOpenGLWindow);
+}
+
+void OpenGLWindow::toggleFullscreen()
+{
+  QSettings settings;
+  if (isFullScreen())
+  {
+    // Show the window normal or maximized (depending on how it was shown before)
+    if (showNormalMaximized)
+      showMaximized();
+    else
+      showNormal();
+  }
+  else
+  {
+    // Save if the window is currently maximized
+    showNormalMaximized = isMaximized();
+
+    showFullScreen();
+  }
+}
+
+
+void OpenGLWindow::keyPressEvent(QKeyEvent *event)
+{
+  int key = event->key();
+  bool controlOnly = (event->modifiers() == Qt::ControlModifier);
+
+  if (key == Qt::Key_Escape)
+  {
+    if (isFullScreen())
+      toggleFullscreen();
+  }
+  else if (key == Qt::Key_F && controlOnly)
+    toggleFullscreen();
+  else if (key == Qt::Key_G && controlOnly)
+    close();
+  else
+  {
+    // pass the event on to the QWidget.
+    QWidget::keyPressEvent(event);
+  }
+}