Merge pull request #22 from IENT/development

* Fix Bug concerning deletion of items (increase performance)
* Add tests for bjontegaard delta
* Fix tests broken due to parsing dialog

.gitignore

@@ -102,3 +102,6 @@ example_simulation_data
 # allow encoder logs in the automatic tests folder
 !src/rdplot/tests/test_logs/examplesForDifferentVersions/*/*.log
 !src/rdplot/tests/test_logs/exampleSimLogDirs/*/*.log
+
+#additional data logs
+datDec/

.travis.yml

@@ -25,7 +25,6 @@ before_script:
 
 script:
  - cd src
- - export RUNNING_AS_UNITTEST=True
  - pytest --cov-report= --cov=rdplot
 
 after_success:

appveyor.yml

@@ -11,7 +11,7 @@ environment:
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - cmd: "%PIP% install cycler matplotlib py pyparsing PyQt5 pytest python-dateutil pytz sip six tabulate mpldatacursor jsonpickle xmltodict matplotlib2tikz pynsist"
+  - cmd: "%PIP% install cycler matplotlib py pyparsing PyQt5 pytest python-dateutil pytz sip six tabulate mpldatacursor jsonpickle xmltodict matplotlib2tikz pynsist Pillow"
   - git clone https://github.com/JensAc/WinPythonModules.git
   - cd WinPythonModules
   - cmd: "%PIP% install numpy-1.13.1+mkl-cp36-cp36m-win32.whl scipy-0.19.1-cp36-cp36m-win32.whl"

setup.py

@@ -181,8 +181,7 @@ def get_version():
     # requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
     install_requires=['cycler', 'matplotlib', 'numpy', 'py', 'pyparsing', 'pyqt5<5.11', 'pytest', 'python-dateutil', 'pytz',
-                      'sip', 'six', 'scipy', 'tabulate', 'mpldatacursor',
-                      'xmltodict', 'jsonpickle', 'matplotlib2tikz'],
+                      'sip', 'six', 'scipy', 'tabulate', 'mpldatacursor', 'xmltodict', 'jsonpickle', 'matplotlib2tikz', 'Pillow'],
 
     # List additional groups of dependencies here (e.g. development
     # dependencies). You can install these using the following syntax,

src/installer.cfg

@@ -32,7 +32,7 @@ packages = rdplot
     jsonpickle
     xmltodict
     matplotlib2tikz
-
+    PIL
 
 # Other files and folders that should be installed
 files = ../LICENSE.txt

src/rdplot/SimulationDataItem.py

@@ -24,6 +24,9 @@
 from copy import copy
 from os import listdir
 from os.path import join, abspath, isfile, isdir
+from PyQt5.QtCore import *
+from PyQt5.QtWidgets import QDialog, QVBoxLayout, QHBoxLayout, QComboBox, QPushButton, QLabel, QCheckBox, QGroupBox, QMessageBox, QApplication
+import re
 
 
 #
@@ -294,7 +297,7 @@ def _is_file_text_matching_re_pattern(cls, path, pattern):
             return bool(re.search(pattern, text, re.M + re.X))
 
 
-class SimulationDataItemFactory:
+class SimulationDataItemFactory(QObject):
     """This class is a factory for all sub classes of the :class:
     `AbstractSimulationDataItem` class.
 
@@ -311,10 +314,12 @@ class SimulationDataItemFactory:
     first matching class will be used to create the item. Thus, more general
     items should be tried at last.
     """
-
+    parsingError = pyqtSignal()
     # Constructors
     def __init__(self, classes=None):
+        super().__init__()
         self._classes = set()
+        self.class_selection_dialog = None
 
         if classes is not None:
             for cls in classes:
@@ -348,6 +353,7 @@ def from_path(cls, directory_path):
 
             # Add all sub classes of AbstractSimulationDataItem from the module
             # to the factory
+            class_list = []
             for module_item in imported_module.__dict__.values():
                 if is_class(module_item):
                     try:
@@ -361,7 +367,7 @@ def from_path(cls, directory_path):
                         pass
 
         # end snippet
-
+        simulation_data_item_factory.class_selection_dialog = ClassSelectionDialog()
         return simulation_data_item_factory
 
     # Interface to Set of Classes
@@ -391,10 +397,31 @@ def create_item_from_file(self, file_path):
         # the file
         cls_list = []
         # try parser, in the order given by their parse_order attribute. use the first one that can parse the file
-        for cls in reversed(sorted(self._classes, key=lambda parser_class: parser_class.parse_order)):
+        list_classes = list(reversed(sorted(self._classes, key=lambda parser_class: parser_class.parse_order)))
+        list_classes = list(filter(lambda x: True if str(x).find('Abstract') == -1 else False, list_classes))
+        for cls in list_classes:
             if cls.can_parse_file(file_path):
                 cls_list.append(cls(file_path))
                 break
+        # checking for parsers automatically would be a lot easier at this point
+        # but user would not be able to manually select a format
+        if not cls_list and isfile(file_path):
+            if not self.class_selection_dialog.get_remember_decision():
+                self.class_selection_dialog.set_items(list(map(lambda x: re.sub('<|>|\'', '', str(x)).split('.')[-1],
+                                                               list_classes)))
+                result = self.class_selection_dialog.exec_(file_path)
+                if result == QDialog.Accepted:
+                    try:
+                        cls_list.append(list_classes[self.class_selection_dialog.get_selected_class()](file_path))
+                    except:
+                        self.parsingError.emit()
+                elif result == QDialog.Rejected and self.class_selection_dialog.get_remember_decision():
+                    raise RuntimeError()
+            else:
+                try:
+                    cls_list.append(list_classes[self.class_selection_dialog.get_selected_class()](file_path))
+                except:
+                    self.parsingError.emit()
         return cls_list
 
     def create_item_list_from_directory(self, directory_path):
@@ -412,14 +439,15 @@ def create_item_list_from_directory(self, directory_path):
             try:
                 item_list.extend(self.create_item_from_file(path))
                 print("Parsed '{}' ".format(path))
+            except RuntimeError:
+                break
             except SimulationDataItemError as error:
                 pass
                 # We definitely cannot accept thousands of exceptions on the command line
                 # print((AbstractEncLog
                 #    "Could not create simulation data item from file '{}'"
                 #    " due to {}"
                 # ).format(path, error))
-
         return item_list
 
     def create_item_list_from_path(self, path):
@@ -432,6 +460,7 @@ def create_item_list_from_path(self, path):
 
         :rtype: :class: `list` of simulation data items
         """
+        self.class_selection_dialog.reset()
 
         if isfile(path):
             return self.create_item_from_file(path)
@@ -452,3 +481,42 @@ def __str__(self):
 
     def __repr__(self):
         return str("SimulationDataItemFactory with loaded classes: ".format(str(self)))
+
+
+class ClassSelectionDialog(QDialog):
+    def __init__(self):
+        super().__init__()
+        self.setWindowTitle('Error!')
+        self.setLayout(QVBoxLayout())
+        self._text_label = QLabel()
+        self.layout().addWidget(self._text_label)
+        self._combo_box = QComboBox()
+        self.layout().addWidget(self._combo_box)
+        self._check_box = QCheckBox('Remember my decision for future errors')
+        self.layout().addWidget(self._check_box)
+        self._group_box = QGroupBox()
+        self._group_box.setLayout(QHBoxLayout())
+        self._button1, self._button2 = QPushButton('Accept'), QPushButton('Cancel')
+        self._group_box.layout().addWidget(self._button1)
+        self._group_box.layout().addWidget(self._button2)
+        self.layout().addWidget(self._group_box)
+        self._button1.clicked.connect(self.accept)
+        self._button2.clicked.connect(self.reject)
+
+    def get_selected_class(self):
+        return self._combo_box.currentIndex()
+
+    def exec_(self, file_name):
+        self._text_label.setText(('Problem with file: {}\nNo matching parsers were found.\nPlease select one of the existing ones or '
+                                       'implement a new parser.').format(file_name.split('/')[-1]))
+        return super().exec_()
+
+    def set_items(self, items):
+        self._combo_box.clear()
+        self._combo_box.addItems(items)
+
+    def get_remember_decision(self):
+        return self._check_box.isChecked()
+
+    def reset(self):
+        self._check_box.setChecked(False)

src/rdplot/Widgets/MainWindow.py

@@ -1,11 +1,12 @@
 from os import path
 from os.path import sep, isfile, isdir
 import csv
+import cProfile, pstats
 
 import pkg_resources
 import jsonpickle
 from PyQt5 import QtWidgets, QtGui
-from PyQt5.QtCore import QItemSelectionModel
+from PyQt5.QtCore import QItemSelectionModel, QItemSelection, QObject
 from PyQt5.uic import loadUiType
 
 
@@ -20,6 +21,19 @@
 here = pkg_resources.resource_filename('rdplot','')
 #here = path.abspath(path.dirname(__file__) + '/../')
 
+def do_cprofile(func):
+    def profiled_func(*args, **kwargs):
+        profile = cProfile.Profile()
+        try:
+            profile.enable()
+            result = func(*args, **kwargs)
+            profile.disable()
+            return result
+        finally:
+            stats = pstats.Stats(profile).sort_stats('cumtime')
+            stats.dump_stats('remove_items_new.profile')
+    return profiled_func
+
 class MainWindow(QMainWindow, Ui_MainWindow):
     def __init__(self, ):
         super(MainWindow, self).__init__()
@@ -180,7 +194,12 @@ def remove(self):
         # List call necessary to avoid runtime error because of elements changing
         # during iteration
         self._variable_tree_selection_model.selectionChanged.disconnect()
+        # disconnect slot to avoid multiple function triggers by selectionChanged signal
+        # not disconnecting slows program down significantly
+        self._selection_model.selectionChanged.disconnect(self.change_list)
         self.simDataItemTreeModel.remove(list(values))
+        self.change_list(QItemSelection(), QItemSelection())
+        self._selection_model.selectionChanged.connect(self.change_list)
         self._variable_tree_selection_model.selectionChanged.connect(self.update_plot)
 
     def change_list(self, q_selected, q_deselected):

src/rdplot/lib/BD.py

@@ -289,6 +289,7 @@ def bjontegaard(curve1, curve2, mode='dsnr', interpol='pol', seq='', d=list(), t
     if mode == 'dsnr':
         return bdsnr(rate1, psnr1, rate2, psnr2, interpol, seq, d, testmode)
     elif mode == 'drate':
+        # this branch will never be executed because of line 273
         if interpol == 'pchip4':
             rate1 = [i[0] for i in curve1]
             rate2 = [i[0] for i in curve2]

src/rdplot/model.py

@@ -730,6 +730,7 @@ class SimDataItemTreeModel(OrderedDictTreeModel):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.dialog = AttributesDialog()
 
     # Implement *add*, *update* and remove to add/remove sim data items to the
     # tree.
@@ -764,6 +765,10 @@ def update(self, sim_data_items, check_add_param = True):
         # if you want to simulate with your own parameters, make sure that they appear in the
         # logfile
         try:
+            self.dialog.chosen_par.clear()
+            self.dialog.not_chosen_par.clear()
+            self.dialog.reset = True
+            QP_added = False
             for sim_data_item in sim_data_items:
                 if sim_data_item.__class__ not in all_log_configs:
                     all_log_configs[sim_data_item.__class__] = []
@@ -777,9 +782,9 @@ def update(self, sim_data_items, check_add_param = True):
                 # no dialog window will be opened where you could add/remove all configuration parameters
                 # that differ between the sim_data_items
                 # TODO: enable multiple configuration parameters for the case that a specific file is opened
-                chosen_par = QtWidgets.QListWidget()
-                if hasattr(sim_data_item, 'qp'):
-                    chosen_par.addItems(['QP'])
+                if hasattr(sim_data_item, 'qp') and not QP_added:
+                    self.dialog.chosen_par.addItems(['QP'])
+                    QP_added = True
                 # print(sim_data_item.summary_data['encoder_config'])
             value_filter = ['.yuv', '.bin', '.hevc', '.jem']
             key_filter = []
@@ -800,37 +805,11 @@ def update(self, sim_data_items, check_add_param = True):
                 # the user can drag the parameters, he wants to analyse further into an other list
                 # the order of the parameters in the list determines the order of the parameter tree
                 if diff_dict[sim_class]:
-                    chosen_par.setDragDropMode(QAbstractItemView.DragDrop)
-                    chosen_par.setDefaultDropAction(QtCore.Qt.MoveAction)
-                    not_chosen_par = QtWidgets.QListWidget()
-                    not_chosen_par.addItems([item for item in diff_dict[sim_class] if item != 'QP'])
-                    not_chosen_par.setDragDropMode(QAbstractItemView.DragDrop)
-                    not_chosen_par.setDefaultDropAction(QtCore.Qt.MoveAction)
-                    if not_chosen_par:
-                        # we do not want to create the dialog when testing the code. since the dialog will never be closed
-                        # todo: code should not know about test. make dialog available from the outside, let test close it
-                        if 'RUNNING_AS_UNITTEST' not in environ:
-                            main_layout = QVBoxLayout()
-                            dialog = QDialog()
-                            dialog.setWindowTitle('Choose Parameters')
-                            dialog.setLayout(main_layout)
-                            msg = QtWidgets.QLabel()
-                            msg.setText('Additional Parameters have been found.\n'
-                                        'Move Parameters you want to consider further to the right list.\n')
-                            main_layout.addWidget(msg)
-                            list_layout = QHBoxLayout()
-                            main_layout.addLayout(list_layout)
-                            # TODO: all items dragged into chosen_par should appear above the qp_item
-                            list_layout.addWidget(not_chosen_par)
-                            list_layout.addWidget(chosen_par)
-                            not_chosen_par.show()
-                            chosen_par.show()
-                            ok_button = QPushButton('OK')
-                            main_layout.addWidget(ok_button)
-                            ok_button.clicked.connect(dialog.close)
-                            dialog.exec()
-                    for i in range(len(not_chosen_par)):
-                        diff_dict[sim_class].pop(not_chosen_par.item(i).text(), None)
+                    self.dialog.not_chosen_par.addItems([item for item in diff_dict[sim_class] if item != 'QP'])
+                    if self.dialog.not_chosen_par:
+                        self.dialog.exec_()
+                    for i in range(len(self.dialog.not_chosen_par)):
+                        diff_dict[sim_class].pop(self.dialog.not_chosen_par.item(i).text(), None)
                 additional_param_found.append(sim_class)
 
         except AttributeError:
@@ -842,7 +821,7 @@ def update(self, sim_data_items, check_add_param = True):
 
             has_additional_params = False
             if sim_data_item.__class__ in additional_param_found:
-                sim_data_item.additional_params = [chosen_par.item(i).text() for i in range(len(chosen_par))]
+                sim_data_item.additional_params = [self.dialog.chosen_par.item(i).text() for i in range(len(self.dialog.chosen_par))]
                 has_additional_params = True
 
             # Get *item* of the tree corresponding to *sim_data_item*
@@ -874,7 +853,6 @@ def remove(self, sim_data_items):
 
         :param sim_data_items: Iterable collection of :class: `SimDataItem`s to be removed
         """
-
         for sim_data_item in sim_data_items:
             # Get *item* of the tree corresponding to *sim_data_item*
             item = self.create_path(*sim_data_item.tree_identifier_list)
@@ -1170,3 +1148,40 @@ class LatexTemplate(Template):
             latex_template = LatexTemplate(template_file.read())
             new_latex_doc = latex_template.substitute(table_goeth_here=latex_table)
             output_file.write(new_latex_doc)
+
+
+class AttributesDialog(QDialog):
+
+    message_shown = pyqtSignal()
+
+    def __init__(self):
+        super().__init__()
+        self.reset = True
+        main_layout = QVBoxLayout()
+        self.setWindowTitle('Choose Parameters')
+        self.setLayout(main_layout)
+        msg = QtWidgets.QLabel()
+        msg.setText('Additional Parameters have been found.\n'
+                    'Move Parameters you want to consider further to the right list.\n')
+        main_layout.addWidget(msg)
+        list_layout = QHBoxLayout()
+        main_layout.addLayout(list_layout)
+        # TODO: all items dragged into chosen_par should appear above the qp_item
+        self.not_chosen_par = QtWidgets.QListWidget()
+        self.chosen_par = QtWidgets.QListWidget()
+        self.chosen_par.setDragDropMode(QAbstractItemView.DragDrop)
+        self.chosen_par.setDefaultDropAction(QtCore.Qt.MoveAction)
+        self.not_chosen_par.setDragDropMode(QAbstractItemView.DragDrop)
+        self.not_chosen_par.setDefaultDropAction(QtCore.Qt.MoveAction)
+        list_layout.addWidget(self.not_chosen_par)
+        list_layout.addWidget(self.chosen_par)
+        self.not_chosen_par.show()
+        self.chosen_par.show()
+        ok_button = QPushButton('OK')
+        main_layout.addWidget(ok_button)
+        ok_button.clicked.connect(self.close)
+
+    def paintEvent(self, event):
+        if self.reset:
+            self.message_shown.emit()
+            self.reset = False