Some automation of routine processes of maintaining videos.

.gitignore

@@ -102,5 +102,6 @@ ENV/
 
 # CUSTOM
 .idea/
+videos/*
+downloads/*
 config.json
-

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.autopep8"
+    },
+    "python.formatting.provider": "none",
+    "githubPullRequests.ignoredPullRequestBranches": [
+        "master"
+    ]
+}

Downloader.py

@@ -4,11 +4,12 @@
 from streamonitor.managers.httpmanager import HTTPManager
 from streamonitor.managers.climanager import CLIManager
 from streamonitor.managers.zmqmanager import ZMQManager
+from streamonitor.managers.dashmanager import DashManager
 from streamonitor.managers.outofspace_detector import OOSDetector
 from streamonitor.clean_exit import CleanExit
 import streamonitor.sites  # must have
 
-        
+
 def is_docker():
     path = '/proc/self/cgroup'
     return (
@@ -36,8 +37,11 @@ def main():
     zmq_manager = ZMQManager(streamers)
     zmq_manager.start()
 
-    http_manager = HTTPManager(streamers)
-    http_manager.start()
+    # http_manager = HTTPManager(streamers)
+    # http_manager.start()
+
+    dash_manager = DashManager(streamers)
+    dash_manager.start()
 
 
 main()

README.md

@@ -1,7 +1,9 @@
 # StreaMonitor
-A Python3 application for monitoring and saving (mostly adult) live streams from various websites.
+A Python application for monitoring and saving (mostly adult) live streams from various websites.
 
-Inspired by [Recordurbate](https://github.com/oliverjrose99/Recordurbate)
+# Credits
+- Inspired by: [Recordurbate](https://github.com/oliverjrose99/Recordurbate)
+- Thumbnail system code by: [VCSI](https://github.com/amietn/vcsi)
 
 ## Supported sites
 | Site name      | Abbreviation | Aliases                     | Quirks                 | Selectable resolution |
@@ -30,8 +32,11 @@ There are hundreds of clones of the sites above, you can read about them on [thi
 
 ## Requirements
 * Python 3
-  * Install packages listed in requirements.txt with pip.
-* FFmpeg
+  * Install packages listed in requirements.txt:
+  ```
+  pip install -r requirements.txt
+  ```
+* [FFmpeg](https://ffmpeg.org/download.html)
 
 ## Usage
 
@@ -40,13 +45,26 @@ The application has the following interfaces:
 * External console via ZeroMQ (sort of working)
 * Web interface (only status)
 
-#### Starting and console
-Start the downloader (it does not fork yet)\
+Here the `config.json` file structure:
+``` json
+[
+    {
+        "site": "StripChat",
+        "username": "name",
+        "running": true
+    },
+]
+```
+
+#### Starting
+Start the downloader with the following command.
 Automatically imports all streamers from the config file.
 ```
-python3 Downloader.py
+python Downloader.py
 ```
+(On Windows you can use the `run.bat` file)
 
+#### Console
 On the console you can use the following commands:
 ```
 add <username> <site> - Add streamer to the list (also starts monitoring)
@@ -67,32 +85,32 @@ For the `site` input, you can use either the full or the short format of the sit
 #### "Remote" controller
 Add or remove a streamer to record (Also saves config file)
 ```
-python3 Controller.py add <username> <website>
-python3 Controller.py remove <username>
+python Controller.py add <username> <website>
+python Controller.py remove <username>
 ```
 
 Start/stop recording streamers
 ```
-python3 Controller.py <start|stop> <username>
+python Controller.py <start|stop> <username>
 ```
 
 List the streamers in the config
 ```
-python3 Controller.py status
+python Controller.py status
 ```
 
 #### Web interface
 
 You can access the web interface on port 5000. 
 It just prints the same information as the status command. 
-You can also get a list of the recorded streams.
+~~You can also get a list of the recorded streams.~~
 
 Further improvements can be expected.
 
-## Docker support
+## ~~Docker support~~
 
-You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use.
-Simply start it in the folder with `docker-compose up`.
+~~You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use.~~
+~~Simply start it in the folder with `docker-compose up`.~~
 
 ## Configuration
 

config/web-config.json

@@ -0,0 +1,12 @@
+{
+    "update_interval_s": 1,
+    "color_positive": "#3D9970",
+    "color_warning": "#FFDC00",
+    "color_negative": "#FF4136",
+    "color_neutral": "#DDDDDD",
+    "color_text": "#EEEEEE",
+    "color_text_inv": "#191919",
+    "color_dropdown_background": "#606060",
+    "color_cell_background": "#2f2f2f",
+    "color_table_header_background": "#101010"
+}

config/web-lexicon.json

@@ -0,0 +1,22 @@
+{
+    "title": "StreaMonitor",
+    "remove": "Remove",
+    "add": "Add",
+    "stop_all": "Stop all",
+    "start_all": "Start all",
+    "username": "Username",
+    "site": "Site",
+    "status": "Status",
+    "started": "Started",
+    "console_placeholder": "Console output will appear here",
+    "loading": "Loading",
+    "running": "Running",
+    "cpu": "CPU",
+    "ram": "RAM",
+    "harddrive": "Disk",
+    "gigabyte": "GB",
+    "limit": "Limit",
+    "credits": "Kseen715@git",
+    "versions": "Plotly-5.14.1 Dash-2.9.3",
+    "v_manager": "V-Manager"
+}

mac.bat

@@ -0,0 +1 @@
+python mac.py

mac.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import colorama as col
+import threading as th
+from tqdm import tqdm
+import make_vcsis
+
+from src import move_all as ma
+from src import splitter as sp
+from src import delete_less as dl
+
+
+def move_and_split(src: str, dst: str, file_size: int = None, log: bool = True,
+                   tqdm_opt: bool = True):
+    """
+    Move all files from subdirectories of src into dst.
+    Then split all video files in dst into file_size chunks
+    and delete all original files.
+
+    Parameters
+    ----------
+        src (str): source directory
+        dst (str): destination directory
+        file_size (int): size of chunks in Bytes, default is None
+    """
+    ma.move_all_subs(src, dst, log=True)
+    ma.delete_empty_dirs(src, log=True)
+    # find all video files in dst
+    files = []
+    for root, dirs, filenames in os.walk(dst):
+        for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\
+[MAC]: Finding video files')\
+                if tqdm_opt else filenames:
+            if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')):
+                files.append(os.path.join(root, filename))
+    # split all video files in dst
+
+    def call_function(filename):
+        try:
+            sp.main_split(filename, split_filesize=file_size, tqdm_opt=False)
+        except Exception as e:
+            print(
+                f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\
+ERROR:{col.Style.RESET_ALL} {e}')
+        except KeyboardInterrupt:
+            print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\
+ERROR:{col.Style.RESET_ALL} Keyboard interrupt')
+
+    threads = []
+    for f in tqdm(files, desc=f'{col.Style.RESET_ALL}[MAC]: Starting threads')\
+            if tqdm_opt else files:
+        # TODO: limit thread count to a CPU count
+        t = th.Thread(target=call_function, args=(f,))
+        threads.append(t)
+        t.start()
+
+    for t in tqdm(threads, desc=f'{col.Style.RESET_ALL}[MAC]: Splitting videos')\
+            if tqdm_opt else threads:
+        t.join()
+    if log:
+        print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.GREEN}\
+DONE!{col.Style.RESET_ALL}')
+
+
+def vcsis(foldername: str = 'videos', overwrite: bool = False):
+    """
+    Create a video contact sheet image for each video file in foldername.
+
+    Parameters
+    ----------
+        foldername (str): name of folder containing video files
+        overwrite (bool): overwrite existing vcsi files, default is False
+    """
+    for root, dirs, filenames in os.walk(foldername):
+        for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\
+[VCSI]: Making thumbnails'):
+            if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')):
+                try:
+                    make_vcsis.main(foldername + '/' +
+                                    filename, overwrite=overwrite)
+                except Exception as e:
+                    print(
+                        f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\
+ERROR:{col.Style.RESET_ALL} {e}')
+                except KeyboardInterrupt:
+                    print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\
+ERROR:{col.Style.RESET_ALL} Keyboard interrupt')
+    print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.GREEN}\
+DONE!{col.Style.RESET_ALL}')
+
+
+if __name__ == '__main__':
+    move_and_split('downloads', 'videos', file_size=2000000000)
+    dl.delete_less('videos', 200000000, log=True)
+    vcsis('videos', overwrite=False)
+    input()

make_vcsis.py

@@ -0,0 +1,16 @@
+import os
+import sys
+import subprocess as sub
+import vcsi
+
+
+def main(filename: str = 'videos', overwrite: bool = False):
+    args = ["vcsi", "-t", "-w 7680", "-g 5x5"]
+    if not overwrite:
+        args.append("--no-overwrite")
+    args.append(filename)
+    cp = sub.run(args)
+
+
+if __name__ == '__main__':
+    main('videos/AbsintheGirl-20230609-121937.mp4')

parameters.py

@@ -1,13 +1,13 @@
 DOWNLOADS_DIR = 'downloads'
-MIN_FREE_DISK_PERCENT = 1.0  # in %
+MIN_FREE_DISK_PERCENT = 10.0  # in %
 DEBUG = False
 
 # You can enter a number to select a specific height.
 # Use a huge number here and closest match to get the highest resolution variant
 # Eg: 240, 360, 480, 720, 1080, 1440, 99999
-WANTED_RESOLUTION = 1080
+WANTED_RESOLUTION = 3840
 
 # Specify match type when specified height
 # Possible values: exact, exact_or_least_higher, exact_or_highest_lower, closest
 # Beware of the exact policy. Nothing gets downloaded if the wanted resolution is not available
-WANTED_RESOLUTION_PREFERENCE = 'closest'
+WANTED_RESOLUTION_PREFERENCE = 'closest'

requirements.txt

@@ -1,9 +1,54 @@
-requests
-terminaltables
-pyzmq
-flask
-termcolor
-beautifulsoup4
-websocket-client
-ffmpy
-m3u8
+backports-datetime-fromisoformat==2.0.1
+beautifulsoup4==4.12.3
+blinker==1.7.0
+cachelib==0.9.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+dash==2.16.1
+dash-bootstrap-components==1.5.0
+dash-core-components==2.0.0
+dash-extensions==1.0.14
+dash-html-components==2.0.0
+dash-table==5.0.0
+dataclass-wizard==0.22.3
+EditorConfig==0.12.4
+ffmpy==0.3.2
+flask==3.0.2
+Flask-Caching==2.1.0
+idna==3.6
+importlib-metadata==7.1.0
+itsdangerous==2.1.2
+Jinja2==3.1.3
+jsbeautifier==1.15.1
+m3u8==4.0.0
+MarkupSafe==2.1.5
+more-itertools==9.1.0
+nest-asyncio==1.6.0
+numpy==1.26.4
+packaging==24.0
+pandas==2.2.1
+parsedatetime==2.6
+pillow==10.2.0
+plotly==5.20.0
+psutil==5.9.8
+python-dateutil==2.9.0.post0
+pytz==2024.1
+pyzmq==25.1.2
+requests==2.31.0
+retrying==1.3.4
+six==1.16.0
+soupsieve==2.5
+tenacity==8.2.3
+termcolor==2.4.0
+terminaltables==3.1.10
+texttable==1.7.0
+tqdm==4.66.2
+typing-extensions==4.10.0
+tzdata==2024.1
+urllib3==2.2.1
+vcsi==7.0.16
+websocket-client==1.7.0
+werkzeug==3.0.1
+zipp==3.18.1

run.bat

@@ -0,0 +1 @@
+python Downloader.py