Merge pull request #55 from netinvent/retention-policy

v3.0.0-beta1

.github/workflows/linux.yaml

@@ -0,0 +1,35 @@
+name: linux-tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        # Python 3.3 and 3.4 have been removed since github won't provide these anymore
+        # As of 2023/01/09, we have removed python 3.5 and 3.6 as they don't work anymore with linux on github
+        # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12", 'pypy-3.6', 'pypy-3.7', 'pypy-3.8', 'pypy-3.10']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        if [ -f npbackup/requirements.txt ]; then pip install -r npbackup/requirements.txt; fi
+    - name: Generate Report
+      env:
+        RUNNING_ON_GITHUB_ACTIONS: true
+      run: |
+        pip install pytest coverage
+        python -m coverage run -m pytest -vvs tests
+    - name: Upload Coverage to Codecov
+      uses: codecov/codecov-action@v3

.github/workflows/pylint-linux.yaml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # python-version: [3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", 'pypy-3.6', 'pypy-3.7']
-        python-version: ["3.11"]
+        python-version: ["3.12"]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,27 +25,27 @@ jobs:
         if [ -f npbackup/requirements.txt ]; then pip install -r npbackup/requirements.txt; fi
         if [ -f upgrade_server/requirements.txt ]; then pip install -r upgrade_server/requirements.txt; fi
     - name: Lint with Pylint
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.11' }}
       run: |
         python -m pip install pylint
         # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist
         # Disable E0401 import error since we lint on linux and pywin32 is obviously missing
-        python -m pylint --disable=C,W,R --max-line-length=127 npbackup
+        python -m pylint --disable=C,W,R --max-line-length=127 --ignore PySimpleGUI.py npbackup
         python -m pylint --disable=C,W,R --max-line-length=127 upgrade_server/upgrade_server
     - name: Lint with flake8
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.11' }}
       run: |
         python -m pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics npbackup
+        python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --exclude PySimpleGUI.py npbackup
         python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics upgrade_server/upgrade_server
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics npbackup
+        python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude PySimpleGUI.py npbackup
         python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics upgrade_server
     - name: Lint with Black
       # Don't run on python < 3.6 since black does not exist there, run only once
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.11' }}
       run: |
         pip install black
-        python -m black --check npbackup
+        python -m black --check --exclude PySimpleGUI.py npbackup
         python -m black --check upgrade_server/upgrade_server

.github/workflows/pylint-windows.yaml

@@ -11,7 +11,7 @@ jobs:
         os: [windows-latest]
         # Don't use pypy on windows since it does not have pywin32 module
         # python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
 
     steps:
     - uses: actions/checkout@v2
@@ -26,26 +26,26 @@ jobs:
         if (Test-Path "npbackup/requirements.txt") { pip install -r npbackup/requirements.txt }
         if (Test-Path "upgrade_server/requirements.txt") { pip install -r upgrade_server/requirements.txt }
     - name: Lint with Pylint
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.12' }}
       run: |
         python -m pip install pylint
         # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist
-        python -m pylint --disable=C,W,R --max-line-length=127 npbackup
+        python -m pylint --disable=C,W,R --max-line-length=127 --ignore PySimpleGUI.py npbackup
         python -m pylint --disable=C,W,R --max-line-length=127 upgrade_server/upgrade_server
     - name: Lint with flake8
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.12' }}
       run: |
         python -m pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics npbackup
+        python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --exclude PySimpleGUI.py npbackup
         python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics upgrade_server/upgrade_server
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics npbackup
+        python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude PySimpleGUI.py npbackup
         python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics upgrade_server/upgrade_server
     - name: Lint with Black
       # Don't run on python < 3.6 since black does not exist there, run only once
-      if: ${{ matrix.python-version == '3.11' }}
+      #if: ${{ matrix.python-version == '3.12' }}
       run: |
         pip install black
-        python -m black --check npbackup
+        python -m black --check --exclude PySimpleGUI.py npbackup
         python -m black --check upgrade_server/upgrade_server

.github/workflows/windows.yaml

@@ -0,0 +1,39 @@
+name: windows-tests
+
+# The default shell here is Powershell
+# Don't run with python 3.3 as using python -m to run flake8 or pytest will fail.
+# Hence, without python -m, pytest will not have it's PYTHONPATH set to current dir and imports will fail
+# Don't run with python 3.4 as github cannot install it (pip install --upgrade pip fails)
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore
+        # Don't test on pypy since we don't have pywin32
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        if (Test-Path "npbackup/requirements.txt") { pip install -r npbackup/requirements.txt }
+    - name: Generate Report
+      env:
+        RUNNING_ON_GITHUB_ACTIONS: true
+      run: |
+        pip install pytest coverage
+        python -m coverage run -m pytest -vvs tests
+    - name: Upload Coverage to Codecov
+      uses: codecov/codecov-action@v3

ANTIVIRUS.md

@@ -0,0 +1,37 @@
+## Antivirus reports from various Nuitka builds for Windows
+
+### 2024/03/10
+
+#### Viewer compilation
+
+Build type: Onefile
+Compiler: Nuitka 2.1 Commercial
+Backend: gcc 13.2.0
+Signed: No
+Build target: npbackup-viewer-x64.exe
+Result: 9/73 security vendors and no sandboxes flagged this file as malicious
+Link: https://www.virustotal.com/gui/file/efb327149ae84878fee9a2ffa5c8c24dbdad2ba1ebb6d383b6a2b23c4d5b723f
+
+Build type: Standalone
+Compiler: Nuitka 2.1 Commercial
+Backend: gcc 13.2.0
+Signed: No
+Build target: npbackup-viewer-x64.exe
+Result: 3/73 security vendors and no sandboxes flagged this file as malicious
+Link: https://www.virustotal.com/gui/file/019ca063a250f79f38582b980f983c1e1c5ad67f36cfe4376751e0473f1fbb29
+
+Build type: Onefile
+Compiler: Nuitka 2.1 Commercial
+Backend: gcc 13.2.0
+Signed: Yes (EV Code signing certificate)
+Build target: npbackup-viewer-x64.exe
+Result: 4/73 security vendors and no sandboxes flagged this file as malicious
+Link: https://www.virustotal.com/gui/file/48211bf5d53fd8c010e60d46aacd04cf4dc889180f9abc6159fde9c6fad65f61
+
+Build type: Standalone
+Compiler: Nuitka 2.1 Commercial
+Backend: gcc 13.2.0
+Signed: Yes (EV Code signing certificate)
+Build target: npbackup-viewer-x64.exe
+Result: 2/73 security vendor and no sandboxes flagged this file as malicious
+Link: https://www.virustotal.com/gui/file/31fbd01a763b25111c879b61c79b5045c1a95d32f02bf2c26aa9a45a9a8583ea

CHANGELOG

@@ -1,6 +1,82 @@
-## Current master
+## Shortly planned
+
+  ! - Add policy like restic forget --keep-within-daily 30d --keep-within-weekly 1m --keep-within-monthly 1y --keep-within-yearly 3y
+  default policy restic forget --keep-within-hourly 72h --keep-within-daily 30d --keep-within-weekly 1m --keep-within-monthly 1y --keep-within-yearly 3y
+      - With button to select --keep-x or --keep-within-x
+      - with button to select forget/prune operation
+      - With explanation link to restic
+    - prune operation: default max unsed ?
+    - Operation planifier:
+      - backups
+      - cleaner (forget / check / prune, each having different planification (at least for check --read-data))
+      - Launch now
+   - NPBackup Operation mode
+      - manages multiple repos with generic key (restic key add) or specified key
+  !- NTP server
+  !- Viewer can have a configuration file
+ ! - Reimplement autoupgrade
+
+## 3.0.0
+ - This is a major rewrite that allows using multiple repositories, adds repository groups and implements repository settings inheritance from group settings
 
-  ! Add --read-concurrency for better backup parallelism on NVME drives (already default to 2 in 0.16.0)
+  !- New operation planifier for backups / cleaning / checking repos
+  !- Implemented retention policies
+  !  - Optional time server update to make sure we don't drift before doing retention operations
+  !  - Optional repo check before doing retention operations
+  !- Pre and post-execution scripts
+  !  - Multiple pre and post execution scripts are now allowed
+  !  - Post-execution script can now be force run on error / exit
+  !  - Script result now has prometheus metrics
+
+
+## Features
+  - New viewer mode allowing to browse/restore restic repositories without any NPBackup configuation
+
+  - Multi repository support
+  - Group settings for repositories
+  !- Operation center
+    - GUI operation center allowing to mass execute actions on repos / groups
+    - CLI operation center via `--group-operation --repo-group=somegroup`
+    !- Implemented retention policies
+    !- Operation planifier allows to create scheduled tasks for operations
+      !- Implemented scheduled task creator for Windows & Unix
+         !(simple list of tasks, actions, stop on error)
+    - Implemented repo quick check / full check / repair index / repair snapshots / unlock / forget / prune / dump / stats commands
+  - Added per repo permission management
+    - Repos now have backup, restore and full privileges, optionally allowing to restrict access for end users
+  - Added snapshot tag to snapshot list on main window
+  - Split npbackup into separate CLI and GUI
+    - Status window has been refactored so GUI now has full stdout / stderr returns from runner and backend
+  - Implemented file size based exclusion
+  - CLI can now fully operate in API compatible mode via --json parameter
+    - Parses non json compliant restic output
+    - Always returns a result boolean and a reason when failing
+  - CLI now accepts --stdin parameter to backup streams sent to CLI
+  - Added minimum backup size upon which we declare that backup has failed
+  - All bytes units now have automatic conversion of units (K/M/G/T/P bits/bytes or IEC bytes)
+  - Refactored GUI and overall UX of configuration
+  - New option --show-config to show inheritance in CLI mode (GUI has visual indicators)
+  - Allow using external restic binary via --external-backend-binary parameter in CLI mode
+
+## Fixes
+  - Default exit code is now worst log level called
+  - Show anonymized repo uri in GUI
+  - Fix deletion failed message for en lang
+  - Fix Google cloud storage backend detection in repository uri
+  - Backup admin password is now stored in a more secure way
+
+## Misc
+  - Current backup state now shows more precise backup state, including last backup date when relevant
+  - Concurrency checks (pidfile checks) are now directly part of the runner
+  - Allow a 30 seconds grace period for child processes to close before asking them nicely, and than not nicely to quit
+  - Fully refactored prometheus metrics parser to be able to read restic standard or json outputs
+  - Added initial tests
+
+## 2.2.2 - 14/12/2023 (internal build only)
+  - Fixed backup paths of '/' root partitions
+  - Properly display repository init errors
+  - Logs didn't show proper error status
+  - Fixed wrong init detection for S3 backend
 
 ## 2.2.1 - 28/08/2023
   - Added snapshot deletion option in GUI

COMPILE_INSTRUCTIONS.md

@@ -79,7 +79,6 @@ The output of the above command should be something like `b'\xa1JP\r\xff\x11u>?V
 Now copy that string into the file `npbackup/secret_keys.py`, which should look like:
 ```
 AES_KEY = b'\xa1JP\r\xff\x11u>?V\x15\xa1\xfd\xaa&tD\xdd\xf9\xde\x07\x93\xd4\xdd\x87R\xd0eb\x10=/'
-DEFAULT_BACKUP_ADMIN_PASSWORD = "MySuperSecretPassword123"
 ```
 
 Note that we also changed the default backup admin password, which is used to see unencrypted configurations in the GUI.