diff --git a/.gitignore b/.gitignore index e35b753..a4c6728 100644 --- a/.gitignore +++ b/.gitignore @@ -102,5 +102,6 @@ ENV/ # CUSTOM .idea/ +videos/* +downloads/* config.json - diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d4379c1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + }, + "python.formatting.provider": "none", + "githubPullRequests.ignoredPullRequestBranches": [ + "master" + ] +} \ No newline at end of file diff --git a/Downloader.py b/Downloader.py index fea95a4..f6be944 100644 --- a/Downloader.py +++ b/Downloader.py @@ -4,11 +4,12 @@ from streamonitor.managers.httpmanager import HTTPManager from streamonitor.managers.climanager import CLIManager from streamonitor.managers.zmqmanager import ZMQManager +from streamonitor.managers.dashmanager import DashManager from streamonitor.managers.outofspace_detector import OOSDetector from streamonitor.clean_exit import CleanExit import streamonitor.sites # must have - + def is_docker(): path = '/proc/self/cgroup' return ( @@ -36,8 +37,11 @@ def main(): zmq_manager = ZMQManager(streamers) zmq_manager.start() - http_manager = HTTPManager(streamers) - http_manager.start() + # http_manager = HTTPManager(streamers) + # http_manager.start() + + dash_manager = DashManager(streamers) + dash_manager.start() main() diff --git a/README.md b/README.md index 992af65..de5b9a2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # StreaMonitor -A Python3 application for monitoring and saving (mostly adult) live streams from various websites. +A Python application for monitoring and saving (mostly adult) live streams from various websites. -Inspired by [Recordurbate](https://github.com/oliverjrose99/Recordurbate) +# Credits +- Inspired by: [Recordurbate](https://github.com/oliverjrose99/Recordurbate) +- Thumbnail system code by: [VCSI](https://github.com/amietn/vcsi) ## Supported sites | Site name | Abbreviation | Aliases | Quirks | Selectable resolution | @@ -30,8 +32,11 @@ There are hundreds of clones of the sites above, you can read about them on [thi ## Requirements * Python 3 - * Install packages listed in requirements.txt with pip. -* FFmpeg + * Install packages listed in requirements.txt: + ``` + pip install -r requirements.txt + ``` +* [FFmpeg](https://ffmpeg.org/download.html) ## Usage @@ -40,13 +45,26 @@ The application has the following interfaces: * External console via ZeroMQ (sort of working) * Web interface (only status) -#### Starting and console -Start the downloader (it does not fork yet)\ +Here the `config.json` file structure: +``` json +[ + { + "site": "StripChat", + "username": "name", + "running": true + }, +] +``` + +#### Starting +Start the downloader with the following command. Automatically imports all streamers from the config file. ``` -python3 Downloader.py +python Downloader.py ``` +(On Windows you can use the `run.bat` file) +#### Console On the console you can use the following commands: ``` add - Add streamer to the list (also starts monitoring) @@ -67,32 +85,32 @@ For the `site` input, you can use either the full or the short format of the sit #### "Remote" controller Add or remove a streamer to record (Also saves config file) ``` -python3 Controller.py add -python3 Controller.py remove +python Controller.py add +python Controller.py remove ``` Start/stop recording streamers ``` -python3 Controller.py +python Controller.py ``` List the streamers in the config ``` -python3 Controller.py status +python Controller.py status ``` #### Web interface You can access the web interface on port 5000. It just prints the same information as the status command. -You can also get a list of the recorded streams. +~~You can also get a list of the recorded streams.~~ Further improvements can be expected. -## Docker support +## ~~Docker support~~ -You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use. -Simply start it in the folder with `docker-compose up`. +~~You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use.~~ +~~Simply start it in the folder with `docker-compose up`.~~ ## Configuration diff --git a/config.json b/config.json deleted file mode 100644 index 0637a08..0000000 --- a/config.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/config/web-config.json b/config/web-config.json new file mode 100644 index 0000000..f40289c --- /dev/null +++ b/config/web-config.json @@ -0,0 +1,12 @@ +{ + "update_interval_s": 1, + "color_positive": "#3D9970", + "color_warning": "#FFDC00", + "color_negative": "#FF4136", + "color_neutral": "#DDDDDD", + "color_text": "#EEEEEE", + "color_text_inv": "#191919", + "color_dropdown_background": "#606060", + "color_cell_background": "#2f2f2f", + "color_table_header_background": "#101010" +} \ No newline at end of file diff --git a/config/web-lexicon.json b/config/web-lexicon.json new file mode 100644 index 0000000..b487bf2 --- /dev/null +++ b/config/web-lexicon.json @@ -0,0 +1,22 @@ +{ + "title": "StreaMonitor", + "remove": "Remove", + "add": "Add", + "stop_all": "Stop all", + "start_all": "Start all", + "username": "Username", + "site": "Site", + "status": "Status", + "started": "Started", + "console_placeholder": "Console output will appear here", + "loading": "Loading", + "running": "Running", + "cpu": "CPU", + "ram": "RAM", + "harddrive": "Disk", + "gigabyte": "GB", + "limit": "Limit", + "credits": "Kseen715@git", + "versions": "Plotly-5.14.1 Dash-2.9.3", + "v_manager": "V-Manager" +} \ No newline at end of file diff --git a/mac.bat b/mac.bat new file mode 100644 index 0000000..7cf84ee --- /dev/null +++ b/mac.bat @@ -0,0 +1 @@ +python mac.py \ No newline at end of file diff --git a/mac.py b/mac.py new file mode 100644 index 0000000..fd9e5cd --- /dev/null +++ b/mac.py @@ -0,0 +1,96 @@ +import os +import sys +import colorama as col +import threading as th +from tqdm import tqdm +import make_vcsis + +from src import move_all as ma +from src import splitter as sp +from src import delete_less as dl + + +def move_and_split(src: str, dst: str, file_size: int = None, log: bool = True, + tqdm_opt: bool = True): + """ + Move all files from subdirectories of src into dst. + Then split all video files in dst into file_size chunks + and delete all original files. + + Parameters + ---------- + src (str): source directory + dst (str): destination directory + file_size (int): size of chunks in Bytes, default is None + """ + ma.move_all_subs(src, dst, log=True) + ma.delete_empty_dirs(src, log=True) + # find all video files in dst + files = [] + for root, dirs, filenames in os.walk(dst): + for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\ +[MAC]: Finding video files')\ + if tqdm_opt else filenames: + if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')): + files.append(os.path.join(root, filename)) + # split all video files in dst + + def call_function(filename): + try: + sp.main_split(filename, split_filesize=file_size, tqdm_opt=False) + except Exception as e: + print( + f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} {e}') + except KeyboardInterrupt: + print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} Keyboard interrupt') + + threads = [] + for f in tqdm(files, desc=f'{col.Style.RESET_ALL}[MAC]: Starting threads')\ + if tqdm_opt else files: + # TODO: limit thread count to a CPU count + t = th.Thread(target=call_function, args=(f,)) + threads.append(t) + t.start() + + for t in tqdm(threads, desc=f'{col.Style.RESET_ALL}[MAC]: Splitting videos')\ + if tqdm_opt else threads: + t.join() + if log: + print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.GREEN}\ +DONE!{col.Style.RESET_ALL}') + + +def vcsis(foldername: str = 'videos', overwrite: bool = False): + """ + Create a video contact sheet image for each video file in foldername. + + Parameters + ---------- + foldername (str): name of folder containing video files + overwrite (bool): overwrite existing vcsi files, default is False + """ + for root, dirs, filenames in os.walk(foldername): + for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\ +[VCSI]: Making thumbnails'): + if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')): + try: + make_vcsis.main(foldername + '/' + + filename, overwrite=overwrite) + except Exception as e: + print( + f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} {e}') + except KeyboardInterrupt: + print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} Keyboard interrupt') + print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.GREEN}\ +DONE!{col.Style.RESET_ALL}') + + +if __name__ == '__main__': + move_and_split('downloads', 'videos', file_size=2000000000) + dl.delete_less('videos', 200000000, log=True) + vcsis('videos', overwrite=False) + input() diff --git a/make_vcsis.py b/make_vcsis.py new file mode 100644 index 0000000..80dd8cc --- /dev/null +++ b/make_vcsis.py @@ -0,0 +1,16 @@ +import os +import sys +import subprocess as sub +import vcsi + + +def main(filename: str = 'videos', overwrite: bool = False): + args = ["vcsi", "-t", "-w 7680", "-g 5x5"] + if not overwrite: + args.append("--no-overwrite") + args.append(filename) + cp = sub.run(args) + + +if __name__ == '__main__': + main('videos/AbsintheGirl-20230609-121937.mp4') diff --git a/parameters.py b/parameters.py index 6dadd4c..971a270 100644 --- a/parameters.py +++ b/parameters.py @@ -1,13 +1,13 @@ DOWNLOADS_DIR = 'downloads' -MIN_FREE_DISK_PERCENT = 1.0 # in % +MIN_FREE_DISK_PERCENT = 10.0 # in % DEBUG = False # You can enter a number to select a specific height. # Use a huge number here and closest match to get the highest resolution variant # Eg: 240, 360, 480, 720, 1080, 1440, 99999 -WANTED_RESOLUTION = 1080 +WANTED_RESOLUTION = 3840 # Specify match type when specified height # Possible values: exact, exact_or_least_higher, exact_or_highest_lower, closest # Beware of the exact policy. Nothing gets downloaded if the wanted resolution is not available -WANTED_RESOLUTION_PREFERENCE = 'closest' \ No newline at end of file +WANTED_RESOLUTION_PREFERENCE = 'closest' diff --git a/requirements.txt b/requirements.txt index f688ab5..2d9de95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,54 @@ -requests -terminaltables -pyzmq -flask -termcolor -beautifulsoup4 -websocket-client -ffmpy -m3u8 \ No newline at end of file +backports-datetime-fromisoformat==2.0.1 +beautifulsoup4==4.12.3 +blinker==1.7.0 +cachelib==0.9.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +dash==2.16.1 +dash-bootstrap-components==1.5.0 +dash-core-components==2.0.0 +dash-extensions==1.0.14 +dash-html-components==2.0.0 +dash-table==5.0.0 +dataclass-wizard==0.22.3 +EditorConfig==0.12.4 +ffmpy==0.3.2 +flask==3.0.2 +Flask-Caching==2.1.0 +idna==3.6 +importlib-metadata==7.1.0 +itsdangerous==2.1.2 +Jinja2==3.1.3 +jsbeautifier==1.15.1 +m3u8==4.0.0 +MarkupSafe==2.1.5 +more-itertools==9.1.0 +nest-asyncio==1.6.0 +numpy==1.26.4 +packaging==24.0 +pandas==2.2.1 +parsedatetime==2.6 +pillow==10.2.0 +plotly==5.20.0 +psutil==5.9.8 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyzmq==25.1.2 +requests==2.31.0 +retrying==1.3.4 +six==1.16.0 +soupsieve==2.5 +tenacity==8.2.3 +termcolor==2.4.0 +terminaltables==3.1.10 +texttable==1.7.0 +tqdm==4.66.2 +typing-extensions==4.10.0 +tzdata==2024.1 +urllib3==2.2.1 +vcsi==7.0.16 +websocket-client==1.7.0 +werkzeug==3.0.1 +zipp==3.18.1 diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..5077a0d --- /dev/null +++ b/run.bat @@ -0,0 +1 @@ +python Downloader.py \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/delete_less.py b/src/delete_less.py new file mode 100644 index 0000000..ace43a2 --- /dev/null +++ b/src/delete_less.py @@ -0,0 +1,62 @@ +import sys +import argparse as arg +import os +import colorama as col + + +def delete_less(directory_path: str, size: int = 200000000, ext: str = ".mp4", + log: bool = True): + """ + Delete all files in directory_path that are less than size in bytes. + + Parameters + ---------- + directory_path (str): path to directory + size (int): size in bytes, default is 200000000 + ext (str): extension of files to delete, default is ".mp4" + log (bool): if True, print all deleted files + """ + counter = 0 + # Get the list of files in the directory + files = os.listdir(directory_path) + # get absolute path + directory_path = os.path.abspath(directory_path) + # Iterate over all the files + for file in files: + # Get the path of the file + file_path = os.path.join(directory_path, file) + # only mp4 + if file_path.endswith(ext): + # Get the size of the file in bytes + size_ = os.path.getsize(file_path) + # If size is less than 200 MB + if size_ < size: + # Delete the file + os.remove(file_path) + if log: + print( + f'{col.Style.RESET_ALL}[DELETER]: \"{file_path}\" \ +deleted') + counter += 1 + if log: + print(f'{col.Style.RESET_ALL}[DELETER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') + + +if __name__ == "__main__": + parser = arg.ArgumentParser(description="Delete files less than 200 MB") + parser.add_argument("-p", "--path", help="Path of the directory") + parser.add_argument("-s", "--size", help="Size of the file") + parser.add_argument("-e", "--ext", help="Extension of the file") + args = parser.parse_args() + # can be only path or path and size or path and size and ext + if args.path and args.size and args.ext: + delete_less(args.path, args.size, args.ext) + elif args.path and args.size: + delete_less(args.path, args.size) + elif args.path: + delete_less(args.path) + else: + print(f"{red}Please provide path of the directory{reset}") + sys.exit(1) + print(f'delete_less :: {green}[SUCCESS]{reset} Done!') diff --git a/src/ffmpeg_split.py b/src/ffmpeg_split.py new file mode 100644 index 0000000..206dcec --- /dev/null +++ b/src/ffmpeg_split.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import csv +import json +import math +import os +import shlex +import subprocess +from optparse import OptionParser +from tqdm import tqdm + + +def split_by_manifest(filename, manifest, vcodec="copy", acodec="copy", + extra="", **kwargs): + """ Split video into segments based on the given manifest file. + + Arguments: + filename (str) - Location of the video. + manifest (str) - Location of the manifest file. + vcodec (str) - Controls the video codec for the ffmpeg video + output. + acodec (str) - Controls the audio codec for the ffmpeg video + output. + extra (str) - Extra options for ffmpeg. + """ + if not os.path.exists(manifest): + print("File does not exist: %s" % manifest) + raise SystemExit + + with open(manifest) as manifest_file: + manifest_type = manifest.split(".")[-1] + if manifest_type == "json": + config = json.load(manifest_file) + elif manifest_type == "csv": + config = csv.DictReader(manifest_file) + else: + print("Format not supported. File must be a csv or json file") + raise SystemExit + + split_cmd = ["ffmpeg", "-i", filename, "-vcodec", vcodec, + "-acodec", acodec, "-y"] + shlex.split(extra) + try: + fileext = filename.split(".")[-1] + except IndexError as e: + raise IndexError("No . in filename. Error: " + str(e)) + for video_config in config: + split_args = [] + try: + split_start = video_config["start_time"] + split_length = video_config.get("end_time", None) + if not split_length: + split_length = video_config["length"] + filebase = video_config["rename_to"] + if fileext in filebase: + filebase = ".".join(filebase.split(".")[:-1]) + + split_args += ["-ss", str(split_start), "-t", + str(split_length), filebase + "." + fileext] + print("########################################################") + print("About to run: " + " ".join(split_cmd + split_args)) + print("########################################################") + subprocess.check_output(split_cmd + split_args) + except KeyError as e: + print("############# Incorrect format ##############") + if manifest_type == "json": + print("The format of each json array should be:") + print("{start_time: , length: , rename_to: }") + elif manifest_type == "csv": + print("start_time,length,rename_to should be the first line ") + print("in the csv file.") + print("#############################################") + print(e) + raise SystemExit + + +def get_video_length(filename): + try: + output = subprocess.check_output(("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", + "default=noprint_wrappers=1:nokey=1", filename)).strip() + video_length = int(float(output)) + # print("Video length in seconds: " + str(video_length)) + except Exception as e: + raise SystemExit + return video_length + + +def ceildiv(a, b): + return int(math.ceil(a / float(b))) + + +def get_name_from_path(path): + return os.path.splitext(os.path.basename(path))[0] + + +def split_by_seconds(filename, split_length, vcodec="copy", acodec="copy", + extra="", video_length=None, tqdm_opt: bool = True, + **kwargs): + if split_length and split_length <= 0: + print("Split length can't be 0") + raise SystemExit + + if not video_length: + video_length = get_video_length(filename) + split_count = ceildiv(video_length, split_length) + if split_count == 1: + # print("Video length is less then the target split length.") + raise SystemExit + + split_cmd = ["ffmpeg", "-i", filename, "-vcodec", + vcodec, "-acodec", acodec] + shlex.split(extra) + try: + filebase = ".".join(filename.split(".")[:-1]) + fileext = filename.split(".")[-1] + except IndexError as e: + raise IndexError("No . in filename. Error: " + str(e)) + for n in tqdm( + range(0, split_count), + desc=f"Splitting {get_name_from_path(filename) + '.' + fileext}")\ + if tqdm_opt else range(0, split_count): + split_args = [] + if n == 0: + split_start = 0 + else: + split_start = split_length * n + + split_args += ["-ss", str(split_start), "-t", str(split_length), + filebase + "-" + str(n + 1) + "-of-" + + str(split_count) + "." + fileext] + # print("About to run: " + " ".join(split_cmd + split_args)) + subprocess.check_output(split_cmd + split_args) + + +def main(): + parser = OptionParser() + + parser.add_option("-f", "--file", + dest="filename", + help="File to split, for example sample.avi", + type="string", + action="store" + ) + parser.add_option("-s", "--split-size", + dest="split_length", + help="Split or chunk size in seconds, for example 10", + type="int", + action="store" + ) + parser.add_option("-c", "--split-chunks", + dest="split_chunks", + help="Number of chunks to split to", + type="int", + action="store" + ) + parser.add_option("-S", "--split-filesize", + dest="split_filesize", + help="Split or chunk size in bytes (approximate)", + type="int", + action="store" + ) + parser.add_option("--filesize-factor", + dest="filesize_factor", + help="with --split-filesize, use this factor in time to" + " size heuristics [default: %default]", + type="float", + action="store", + default=0.95 + ) + parser.add_option("--chunk-strategy", + dest="chunk_strategy", + help="with --split-filesize, allocate chunks according to" + " given strategy (eager or even)", + type="choice", + action="store", + choices=['eager', 'even'], + default='eager' + ) + parser.add_option("-m", "--manifest", + dest="manifest", + help="Split video based on a json manifest file. ", + type="string", + action="store" + ) + parser.add_option("-v", "--vcodec", + dest="vcodec", + help="Video codec to use. ", + type="string", + default="copy", + action="store" + ) + parser.add_option("-a", "--acodec", + dest="acodec", + help="Audio codec to use. ", + type="string", + default="copy", + action="store" + ) + parser.add_option("-e", "--extra", + dest="extra", + help="Extra options for ffmpeg, e.g. '-e -threads 8'. ", + type="string", + default="", + action="store" + ) + (options, args) = parser.parse_args() + + def bailout(): + parser.print_help() + raise SystemExit + + if not options.filename: + bailout() + + if options.manifest: + split_by_manifest(**options.__dict__) + else: + video_length = None + if not options.split_length: + video_length = get_video_length(options.filename) + file_size = os.stat(options.filename).st_size + split_filesize = None + if options.split_filesize: + split_filesize = int( + options.split_filesize * options.filesize_factor) + if split_filesize and options.chunk_strategy == 'even': + options.split_chunks = ceildiv(file_size, split_filesize) + if options.split_chunks: + options.split_length = ceildiv( + video_length, options.split_chunks) + if not options.split_length and split_filesize: + options.split_length = int( + split_filesize / float(file_size) * video_length) + if not options.split_length: + bailout() + split_by_seconds(video_length=video_length, **options.__dict__) + + +if __name__ == '__main__': + main() diff --git a/src/move_all.py b/src/move_all.py new file mode 100644 index 0000000..5b7be92 --- /dev/null +++ b/src/move_all.py @@ -0,0 +1,88 @@ +import os +import sys +import colorama as col + + +def move_all_subs(src: str, dst: str, ext: str = None, log: bool = False): + """ + Move all files from subdirectories of src into dst. + + Parameters + ---------- + src (str): source directory + dst (str): destination directory + ext (str): extension of files to move, default is None + log (bool): if True, print all moved files + """ + if not os.path.isdir(src): + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} \ +\"{src}\" is not a directory') + if not os.path.isdir(dst): + os.mkdir(dst) + + for root, dirs, files in os.walk(src): + for f in files: + if ext is None or f.endswith(ext): + src_path = os.path.join(root, f) + dst_path = os.path.join(dst, f) + try: + # check if file occupied by another process + if os.access(src_path, os.W_OK): + os.rename(src_path, dst_path) + if log: + print(f'{col.Style.RESET_ALL}[MOVER]: \ +\"{src_path}\" -> \"{dst_path}\"') + except PermissionError: + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR{col.Style.RESET_ALL}: \"{src_path}\" is occupied by another process') + except FileExistsError: + print( + f'{col.Style.RESET_ALL}[MOVER]: \ +{col.Fore.RED}ERROR{col.Style.RESET_ALL}: \ +\"{dst_path}\" already exists') + continue + print(f'{col.Style.RESET_ALL}[MOVER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') + + +def delete_empty_dirs(src: str, log: bool = False): + """ + Delete all empty subdirectories of src. + + Parameters + ---------- + src (str): source directory + log (bool): if True, print all deleted directories + """ + if not os.path.isdir(src): + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} \ +\"{src}\" is not a directory') + sys.exit(1) + + for root, dirs, files in os.walk(src, topdown=False): + for d in dirs: + d_path = os.path.join(root, d) + if not os.listdir(d_path): + try: + os.rmdir(d_path) + if log: + print(f'{col.Style.RESET_ALL}[DELETER]: \"{d_path}\" \ +deleted') + except OSError: + print( + f'{col.Style.RESET_ALL}[DELETER]: {col.Fore.RED}\ +ERROR{col.Style.RESET_ALL}: \ +\"{d_path}\" is not empty') + continue + print(f'{col.Style.RESET_ALL}[DELETER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') + + +if __name__ == '__main__': + move_all_subs('downloads', 'videos', ext='.mp4', log=True) + delete_empty_dirs('downloads', log=True) diff --git a/src/splitter.py b/src/splitter.py new file mode 100644 index 0000000..a151672 --- /dev/null +++ b/src/splitter.py @@ -0,0 +1,100 @@ +import os +import sys +import colorama as col +from . import ffmpeg_split as ffs + + +def main_split(filename: str, tqdm_opt: bool = True, split_filesize: int = None, + filesize_factor: float = 0.95, chunk_strategy: str = 'eager', + split_chunks: int = None, split_length: int = None, + vcodec: str = 'copy', acodec: str = 'copy', + extra: str = '-hide_banner -loglevel quiet -y'): + """ + Split video based on a set of options. + + Parameters + ---------- + filename (str): Path to the video file. + tqdm_opt (bool): Show progress bar. Default is True. + split_filesize (int): Split video based on a target filesize in bytes. + filesize_factor (float): Factor to apply to the filesize. \ +Default is 1.0. + chunk_strategy (str): Strategy to use when splitting by filesize. \ +Default is 'even'. + split_chunks (int): Split video into a set number of chunks. + split_length (int): Split video into chunks of a set length in seconds. + vcodec (str): Video codec to use. Default is 'copy'. + acodec (str): Audio codec to use. Default is 'copy'. + extra (str): Extra options for ffmpeg, e.g. '-e -threads 8'. \ +Default is ''. + """ + video_length = ffs.get_video_length(filename) + file_size = os.stat(filename).st_size + split_filesize = int(split_filesize * filesize_factor) + if file_size < split_filesize * 1.2: + return + if split_filesize and chunk_strategy == 'even': + split_chunks = ffs.ceildiv(file_size, split_filesize) + if split_chunks != None: + split_length = ffs.ceildiv(video_length, split_chunks) + if not split_length and split_filesize: + split_length = int( + split_filesize / float(file_size) * video_length) + try: + ffs.split_by_seconds( + video_length=video_length, tqdm_opt=tqdm_opt, + **craft_options(filename, split_filesize, filesize_factor, + chunk_strategy, split_chunks, split_length, + vcodec, acodec, extra)) + os.remove(filename) + except KeyboardInterrupt: + print(col.Fore.RED + 'Aborted by user.' + col.Style.RESET_ALL) + sys.exit(1) + + +def craft_options(filename: str, split_filesize: int = None, + filesize_factor: float = 1.0, chunk_strategy: str = '', + split_chunks: int = None, split_length: int = None, + vcodec: str = 'copy', acodec: str = 'copy', extra: str = '', + manifest: str = None): + """ + Craft options for ffmpeg_split.split_by_seconds(). + + Parameters + ---------- + filename (str): Path to the video file. + split_filesize (int): Split video based on a target filesize in bytes. + filesize_factor (float): Factor to apply to the filesize. \ +Default is 1.0. + chunk_strategy (str): Strategy to use when splitting by filesize. \ +Default is 'even'. + split_chunks (int): Split video into a set number of chunks. + split_length (int): Split video into chunks of a set length in seconds. + vcodec (str): Video codec to use. Default is 'copy'. + acodec (str): Audio codec to use. Default is 'copy'. + extra (str): Extra options for ffmpeg, e.g. '-e -threads 8'. \ +Default is ''. + manifest (str): Split video based on a json manifest file. + + Returns + ------- + options (dict): Options for ffmpeg_split.split_by_seconds(). + """ + options = { + 'filename': filename, + 'split_filesize': split_filesize, + 'filesize_factor': filesize_factor, + 'chunk_strategy': chunk_strategy, + 'split_chunks': split_chunks, + 'split_length': split_length, + 'vcodec': vcodec, + 'acodec': acodec, + 'extra': extra, + 'manifest': manifest + } + return options + + +if __name__ == '__main__': + main_split('./videos/2023-05-22 08-14-39.mp4', + split_filesize=500000000) diff --git a/streamonitor/managers/assets/app.css b/streamonitor/managers/assets/app.css new file mode 100644 index 0000000..9b705a4 --- /dev/null +++ b/streamonitor/managers/assets/app.css @@ -0,0 +1,695 @@ +/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */ + +/* Document + ========================================================================== */ + +/** + * 1. Correct the line height in all browsers. + * 2. Prevent adjustments of font size after orientation changes in iOS. + */ + +html { + line-height: 1.15; + /* 1 */ + -webkit-text-size-adjust: 100%; + /* 2 */ +} + +/* Sections + ========================================================================== */ + +/** + * Remove the margin in all browsers. + */ + +body { + margin: 0; +} + +/** + * Render the `main` element consistently in IE. + */ + +main { + display: block; +} + +/** + * Correct the font size and margin on `h1` elements within `section` and + * `article` contexts in Chrome, Firefox, and Safari. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +/* Grouping content + ========================================================================== */ + +/** + * 1. Add the correct box sizing in Firefox. + * 2. Show the overflow in Edge and IE. + */ + +hr { + box-sizing: content-box; + /* 1 */ + height: 0; + /* 1 */ + overflow: visible; + /* 2 */ +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +pre { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 1em; + /* 2 */ +} + +/* Text-level semantics + ========================================================================== */ + +/** + * Remove the gray background on active links in IE 10. + */ + +a { + background-color: transparent; +} + +/** + * 1. Remove the bottom border in Chrome 57- + * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. + */ + +abbr[title] { + border-bottom: none; + /* 1 */ + text-decoration: underline; + /* 2 */ + text-decoration: underline dotted; + /* 2 */ +} + +/** + * Add the correct font weight in Chrome, Edge, and Safari. + */ + +b, +strong { + font-weight: bolder; +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +code, +kbd, +samp { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 1em; + /* 2 */ +} + +/** + * Add the correct font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` elements from affecting the line height in + * all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sub { + bottom: -0.25em; +} + +sup { + top: -0.5em; +} + +/* Embedded content + ========================================================================== */ + +/** + * Remove the border on images inside links in IE 10. + */ + +img { + border-style: none; +} + +/* Forms + ========================================================================== */ + +/** + * 1. Change the font styles in all browsers. + * 2. Remove the margin in Firefox and Safari. + */ + +button, +input, +optgroup, +select, +textarea { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 100%; + /* 1 */ + line-height: 1.15; + /* 1 */ + margin: 0; + /* 2 */ +} + +/** + * Show the overflow in IE. + * 1. Show the overflow in Edge. + */ + +button, +input { + /* 1 */ + overflow: visible; +} + +/** + * Remove the inheritance of text transform in Edge, Firefox, and IE. + * 1. Remove the inheritance of text transform in Firefox. + */ + +button, +select { + /* 1 */ + text-transform: none; +} + +/** + * Correct the inability to style clickable types in iOS and Safari. + */ + +button, +[type="button"], +[type="reset"], +[type="submit"] { + -webkit-appearance: button; +} + +/** + * Remove the inner border and padding in Firefox. + */ + +button::-moz-focus-inner, +[type="button"]::-moz-focus-inner, +[type="reset"]::-moz-focus-inner, +[type="submit"]::-moz-focus-inner { + border-style: none; + padding: 0; +} + +/** + * Restore the focus styles unset by the previous rule. + */ + +button:-moz-focusring, +[type="button"]:-moz-focusring, +[type="reset"]:-moz-focusring, +[type="submit"]:-moz-focusring { + outline: 1px dotted ButtonText; +} + +/** + * Correct the padding in Firefox. + */ + +fieldset { + padding: 0.35em 0.75em 0.625em; +} + +/** + * 1. Correct the text wrapping in Edge and IE. + * 2. Correct the color inheritance from `fieldset` elements in IE. + * 3. Remove the padding so developers are not caught out when they zero out + * `fieldset` elements in all browsers. + */ + +legend { + box-sizing: border-box; + /* 1 */ + color: inherit; + /* 2 */ + display: table; + /* 1 */ + max-width: 100%; + /* 1 */ + padding: 0; + /* 3 */ + white-space: normal; + /* 1 */ +} + +/** + * Add the correct vertical alignment in Chrome, Firefox, and Opera. + */ + +progress { + vertical-align: baseline; +} + +/** + * Remove the default vertical scrollbar in IE 10+. + */ + +textarea { + overflow: auto; +} + +/** + * 1. Add the correct box sizing in IE 10. + * 2. Remove the padding in IE 10. + */ + +[type="checkbox"], +[type="radio"] { + box-sizing: border-box; + /* 1 */ + padding: 0; + /* 2 */ +} + +/** + * Correct the cursor style of increment and decrement buttons in Chrome. + */ + +[type="number"]::-webkit-inner-spin-button, +[type="number"]::-webkit-outer-spin-button { + height: auto; +} + +/** + * 1. Correct the odd appearance in Chrome and Safari. + * 2. Correct the outline style in Safari. + */ + +[type="search"] { + -webkit-appearance: textfield; + /* 1 */ + outline-offset: -2px; + /* 2 */ +} + +/** + * Remove the inner padding in Chrome and Safari on macOS. + */ + +[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * 1. Correct the inability to style clickable types in iOS and Safari. + * 2. Change font properties to `inherit` in Safari. + */ + +::-webkit-file-upload-button { + -webkit-appearance: button; + /* 1 */ + font: inherit; + /* 2 */ +} + +/* Interactive + ========================================================================== */ + +/* + * Add the correct display in Edge, IE 10+, and Firefox. + */ + +details { + display: block; +} + +/* + * Add the correct display in all browsers. + */ + +summary { + display: list-item; +} + +/* Misc + ========================================================================== */ + +/** + * Add the correct display in IE 10+. + */ + +template { + display: none; +} + +/** + * Add the correct display in IE 10. + */ + +[hidden] { + display: none; +} + +/* END OF RESET.CSS */ + +/* START OF STYLE.CSS */ + +:root { + --color-green-light: #50c590; + --color-green: #3D9970; + --color-green-dark: #2b6a4d; + --color-yellow-light: #ffe53a; + --color-yellow: #FFDC00; + --color-yellow-dark: #bba200; + --color-red-light: #e46c65; + --color-red: #FF4136; + --color-red-dark: #c0352d; + + --color-white-light: #FFFFFF; + --color-white: #EEEEEE; + --color-grey-light: #a1a1a1; + --color-grey: #606060; + --color-grey-dark: #2f2f2f; + --color-black: #191919; + --color-black-dark: #101010; + + --color-background-light: var(--color-grey); + --color-background: var(--color-black); + --color-background-dark: var(--color-black-dark); + --color-text: var(--color-white); + --color-border: var(--color-grey-light); + + --font-jet-brains: JetBrains Mono, monospace; + + --color-shadow: rgba(108, 108, 108, 0.04); + --shadow: + 2px 1px 1px var(--color-shadow), + 2px 2px 2px var(--color-shadow), + 2px 3px 3px var(--color-shadow), + 2px 4px 4px var(--color-shadow), + 2px 5px 5px var(--color-shadow), + 2px 6px 6px var(--color-shadow), + 2px 7px 7px var(--color-shadow); +} + +div#main_div { + margin-left: 15%; + margin-right: 15%; + margin-top: 0; + margin-bottom: 0; + + font-family: var(--font-jet-brains); +} + +html { + box-sizing: border-box; + font-family: var(--jet-brains-font); + background-color: var(--color-background); + color: var(--color-text); +} + +h1 { + text-align: center; +} + +h2 { + text-align: center; +} + +h3 { + margin-top: 0; + margin-bottom: 5px; + text-align: left; +} + +h4 { + margin-top: 0; + text-align: left; +} + +button, +a#v_manager.btn.btn-primary { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 10px; + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; + + text-align: right; + + padding-top: 7px; + padding-bottom: 7px; + padding-left: 20px; + padding-right: 20px; + + min-width: 150px; + min-height: 40px; + + text-align: center; + + background-color: var(--color-background); + color: var(--color-text); + + box-shadow: var(--shadow); +} + +button:hover, +a#v_manager.btn.btn-primary { + background: var(--color-background-dark); + color: var(--color-text); +} + +button#start_all, +button#add_streamer { + background-color: var(--color-green); +} + +a#v_manager.btn.btn-primary { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; + + width: auto; + min-width: 106px; + max-width: 106px; + min-height: 22px; + max-height: 22px; + + background-color: var(--color-black); + + + display: inline-block; + +} + +a#v_manager.btn.btn-primary:hover { + background: var(--color-black-dark); +} + +button#start_all:hover, +button#add_streamer:hover { + background: var(--color-green-dark); +} + +button#stop_all, +button#remove_streamer { + background-color: var(--color-red); +} + +button#stop_all:hover, +button#remove_streamer:hover { + background: var(--color-red-dark); +} + +div#buttons-all-streamers, +div#buttons-single-streamer { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; +} + +div#table { + border-collapse: collapse; + border-spacing: 0; + margin-top: 30px; + margin-bottom: 300px; + + background-color: var(--color-background-light); + box-shadow: var(--shadow); + color: var(--color-text); + font-family: var(--font-jet-brains); +} + +/* class column-header-name */ +.column-header-name { + /* center text */ + text-align: center; +} + +.dash-header { + background-color: var(--color-background-dark); +} + + +input { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; + + background-color: var(--color-background-light); + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +input#username { + width: 300px; + height: 36px; + max-height: 36px; + min-height: 36px; + text-align: left; + + margin-right: 10px; + + color: var(--color-white); + font-family: var(--font-jet-brains); + + padding-left: 10px; + padding-right: 10px; + + box-shadow: var(--shadow); +} + +::placeholder { + color: var(--color-grey-light); +} + +div#user-manage { + display: grid; + /* put elements to the left */ + grid-template-columns: 0fr 0fr; + grid-template-rows: 1fr 1fr; + + margin: 0; + padding: 0; + margin-top: 40px; + margin-bottom: 10px; + height: 40px; + text-align: left; + width: auto; + min-width: 400px; + font-family: var(--font-jet-brains); +} + +div#console_div { + margin-top: 20px; + margin-bottom: 30px; + max-height: 200px; + min-height: 200px; + + padding: 10px; + + background-color: var(--color-background-dark); + color: var(--color-text); + + font-family: var(--font-jet-brains); + box-shadow: var(--shadow); + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +div#credits, +div#versions { + margin-top: 20px; + margin-bottom: 20px; + text-align: center; + font-family: var(--font-jet-brains); +} + +div#credit-version-box { + background-color: var(--color-background-dark); + + padding-top: 20px; + padding-left: 20px; + padding-right: 20px; + padding-bottom: 30px; + + border-color: var(--color-border); + border-width: 2px; + border-left: 0; + border-right: 0; + border-bottom: 0; + border-style: inset; + + box-shadow: var(--shadow); +} + +div#site.dash-dropdown { + margin-top: 0; + margin-bottom: 0; + margin-left: 20px; + margin-right: 0; + + background-color: var(--color-background-light); + min-height: 40px; + max-height: 40px; + height: 40px; + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +div.Select-control { + margin-left: 0; + margin-right: 0; + margin-top: 0; + margin-bottom: 0; +} + +div.Select { + margin-left: 0; + margin-right: 0; + margin-top: 0; + margin-bottom: 0; +} \ No newline at end of file diff --git a/streamonitor/managers/climanager.py b/streamonitor/managers/climanager.py index f946276..c78ef7e 100644 --- a/streamonitor/managers/climanager.py +++ b/streamonitor/managers/climanager.py @@ -2,6 +2,7 @@ from streamonitor.manager import Manager from streamonitor.clean_exit import CleanExit import streamonitor.log as log +import colorama as color if sys.platform != "win32": import readline @@ -10,7 +11,8 @@ class CLIManager(Manager): def __init__(self, streamers): super().__init__(streamers) - self.logger = log.Logger("manager_cli") + self.logger = log.Logger(f"{color.Style.RESET_ALL}{color.Fore.CYAN}\ +manager_cli{color.Style.RESET_ALL}") def run(self): while True: diff --git a/streamonitor/managers/dash_pages/home.py b/streamonitor/managers/dash_pages/home.py new file mode 100644 index 0000000..5c93da8 --- /dev/null +++ b/streamonitor/managers/dash_pages/home.py @@ -0,0 +1,286 @@ +import dash +from dash import Dash +from dash import dcc +from dash import html +from dash import dash_table + +import dash_bootstrap_components as dbc +from streamonitor.managers.dashmanager import lex +from streamonitor.managers.dashmanager import config +from streamonitor.managers.dashmanager import gl_status + +dash.register_page(__name__, + name='StreaMonitor', + path='/', + order=0) + +layout = html.Div([ + html.Div( + [ + html.H1(lex['title']), + # TODO: new buttons: delete all downloads, delete all videos + html.Div( + [ + html.Button( + lex['stop_all'], + id='stop_all', + n_clicks=0 + ), + html.Button( + lex['start_all'], + id='start_all', + n_clicks=0 + ), + dbc.Button( + lex['v_manager'], + id='v_manager', + href='/video-manager', + n_clicks=0 + ), + ], + id="buttons-all-streamers" + ), + # field to enter username + html.Div(children=[ + dcc.Input( + id="username", + type="text", + placeholder=lex['username'], + value="", + style={ + 'max-height': '36px', + 'height': '36px', + }, + ), + dcc.Dropdown( + id="site", + options=[ + {'label': html.Span(['Amateur.TV'], style={ + 'color': config["color_text_inv"]}), + 'value': 'ATV'}, + + {'label': html.Span(['BongaCams'], style={ + 'color': config["color_text_inv"]}), + 'value': 'BC'}, + + {'label': html.Span(['Cam4'], style={ + 'color': config["color_text_inv"]}), + 'value': 'C4'}, + + {'label': html.Span(['Cams.com'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CC'}, + + {'label': html.Span(['CamSoda'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CS'}, + + {'label': html.Span(['Chaturbate'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CB'}, + + {'label': html.Span(['Cherry.TV'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CHTV'}, + + {'label': html.Span(['Dreamcam VR'], style={ + 'color': config["color_text_inv"]}), + 'value': 'DCVR'}, + + {'label': html.Span(['Flirt4Free'], style={ + 'color': config["color_text_inv"]}), + 'value': 'F4F'}, + + {'label': html.Span(['ManyVids Live'], style={ + 'color': config["color_text_inv"]}), + 'value': 'MV'}, + + {'label': html.Span(['MyFreeCams'], style={ + 'color': config["color_text_inv"]}), + 'value': 'MFC'}, + + {'label': html.Span(['SexChat.hu'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SCHU'}, + + {'label': html.Span(['StreaMate'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SM'}, + + {'label': html.Span(['StripChat'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SC'}, + + {'label': html.Span(['StripChat VR'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SCVR'}, + ], + className="site-dropdown", + placeholder=lex['site'], + style={ + 'width': '100px', + 'min-height': '30px', + 'height': '40px', + 'margin': 0, + 'padding': 0, + # 'margin-left': '20px', + 'width': '200px', + 'border-radius': '10px', + 'text-align': 'left', + 'font-family': 'font-family: JetBrains Mono, monospace', + 'background-color': config["color_dropdown_background"], + }, + ) + ], + id="user-manage" + ), + + html.Div( + [ + html.Button( + lex['remove'], + id='remove_streamer', + n_clicks=0 + ), + html.Button( + lex['add'], + id='add_streamer', + n_clicks=0 + ), + ], + id="buttons-single-streamer" + ), + + # console output text box + html.Div( + [ + html.Pre( + id='console', + children=lex['console_placeholder'], + ) + ], + id="console_div" + ), + + html.H3('{} . . .'.format( + lex['loading']), id="running"), + html.H3('{} . . .'.format( + lex['loading']), id="cpu_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="ram_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="disk_space"), + + dcc.Interval( + id='interval-component', + # in milliseconds + interval=config["update_interval_s"]*1000, + n_intervals=0 + ), + # sorting fucntionalities + dash_table.DataTable( + id='table', + columns=[{"name": i, "id": i} + for i in gl_status.columns], + data=gl_status.to_dict('records'), + style_cell={ + 'textAlign': 'left', + 'backgroundColor': config["color_cell_background"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + style_data_conditional=[ + { + 'if': { + 'filter_query': '{Status} = "Channel online"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_positive"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Private show"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Not running"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Rate limited"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Unknown error"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Nonexistent user"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Error on downloading"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream for a while"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_neutral"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_neutral"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + } + ], + style_header={ + 'backgroundColor': config["color_table_header_background"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace', + 'fontWeight': 'bold' + }, + sort_mode="multi", + sort_action="native", + filter_action="native"), + ], + id="main_div"), +]) diff --git a/streamonitor/managers/dash_pages/v_manager.py b/streamonitor/managers/dash_pages/v_manager.py new file mode 100644 index 0000000..f1edebd --- /dev/null +++ b/streamonitor/managers/dash_pages/v_manager.py @@ -0,0 +1,17 @@ +import dash +from dash import Dash +from dash import dcc +from dash import html +from dash import dash_table +from dash import dependencies + + +dash.register_page(__name__, + name='SM - Video Manager', + path='/video-manager', + order=1) + +layout = html.Div([ + html.H1('Video Manager'), + html.Div('This page is not implemented yet.'), +]) diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py new file mode 100644 index 0000000..ac3ef8a --- /dev/null +++ b/streamonitor/managers/dashmanager.py @@ -0,0 +1,176 @@ +from flask import Flask, request +import os +from streamonitor.bot import Bot +import streamonitor.log as log +from streamonitor.manager import Manager +import pandas as pd +import datetime + +import dash +from dash import Dash, dcc, callback, html, dash_table, dependencies +# from dash import Input +# from dash import Output +# from dash import State + +from dash_extensions.enrich import DashProxy, Input, Output, \ + html, MultiplexerTransform, State + +from timeit import default_timer as timer +import time +import shutil +import parameters +import psutil + +from streamonitor.managers.jsonmanager import JsonWorker + + +# load lexicon +lex = JsonWorker.load("./config/web-lexicon.json") +# load config +config = JsonWorker.load("./config/web-config.json") +# null df +gl_status = pd.DataFrame( + columns=[lex['site'], lex['username'], + lex['started'], lex['status']]) + + +app = DashProxy(__name__, + prevent_initial_callbacks=True, + transforms=[MultiplexerTransform()], + title=lex['title'], update_title=None, + external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", + ], + use_pages=True, pages_folder="dash_pages") + +app.css.config.serve_locally = False + + +class DashManager(Manager): + def __init__(self, streamers): + super().__init__(streamers) + self.logger = log.Logger("manager") + + def run(self): + def scripts(): + pass + + def status(): + df = pd.DataFrame( + columns=[lex['site'], lex['username'], + lex['started'], lex['status']]) + for streamer in self.streamers: + df = df._append({lex['site']: streamer.site, + lex['username']: streamer.username, + lex['started']: streamer.running, + lex['status']: streamer.status()}, + ignore_index=True) + + gl_status = df + return df + + status() + + app.layout = html.Div( + [ + dash.page_container, + html.Div( + [ + html.Div( + html.A(lex['credits']), + id="credits", + ), + html.Div( + html.A(lex['versions']), + id="versions", + ), + ], + id="credit-version-box", + ), + ] + ) + + @app.callback( + dependencies.Output('table', 'data'), + dependencies.Input('interval-component', 'n_intervals')) + def update_data(timestamp): + # update table + # status() + return status().to_dict('records') + + @app.callback( + dependencies.Output('running', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_running(timestamp): + # update running + return '=> {}: {}/{}'.\ + format(lex['running'], + len(status()[status()["Status"] == + "Channel online"]), len(status())) + + @app.callback( + dependencies.Output('cpu_usage', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_cpu_usage(timestamp): + # update cpu usage + return '=> {}: {}%'.format(lex['cpu'], psutil.cpu_percent()) + + @app.callback( + dependencies.Output('ram_usage', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_ram_usage(timestamp): + # update ram usage + return '=> {}: {}/{} GB'.\ + format(lex['ram'], + round(psutil.virtual_memory()[3]/1024/1024/1024, 2), + round(psutil.virtual_memory()[0]/1024/1024/1024, 2)) + + @app.callback( + dependencies.Output('disk_space', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_disk_space(timestamp): + # update disk space + return '=> {}: {}/{} {} ({} = {} {})'.\ + format(lex['harddrive'], + round(shutil.disk_usage("./")[1]/1024/1024/1024, 2), + round(shutil.disk_usage("./")[0]/1024/1024/1024, 2), + lex['gigabyte'], + lex['limit'], + round(shutil.disk_usage("./")[0]/1024 * + (100-parameters.MIN_FREE_DISK_PERCENT) / + 1024/1024/100, 2), + lex['gigabyte']) + + @app.callback( + Output('console', 'children'), + Input('stop_all', 'n_clicks')) + def update_stop_all(): + reply = self.execCmd('stop *') + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('start_all', 'n_clicks')) + def update_start_all(): + reply = self.execCmd('start *') + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('remove_streamer', 'n_clicks'), + State('username', 'value')) + def update_remove_streamer(n_clicks, username): + reply = self.execCmd('remove ' + username) + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('add_streamer', 'n_clicks'), + State('username', 'value'), + State('site', 'value')) + def update_add_streamer(n_clicks, username, site): + reply = self.execCmd('add ' + username + ' ' + site) + return datetime.datetime.now().__str__() + ' >> ' + reply + + # dose_of_sex + + app.run(host="127.0.0.1", port=5001) diff --git a/streamonitor/managers/jsonmanager.py b/streamonitor/managers/jsonmanager.py new file mode 100644 index 0000000..9c7326b --- /dev/null +++ b/streamonitor/managers/jsonmanager.py @@ -0,0 +1,17 @@ +import json + + +class JsonWorker: + def load(filepath: str): + ''' + Loads json file from filepath as dict + ''' + with open(filepath, "r") as f: + return json.load(f) + + def save(filepath: str, data: dict): + ''' + Saves data as json file to filepath + ''' + with open(filepath, "w") as f: + json.dump(data, f) diff --git a/streamonitor/sites/chaturbate.py b/streamonitor/sites/chaturbate.py index 9891d5d..4768b9b 100644 --- a/streamonitor/sites/chaturbate.py +++ b/streamonitor/sites/chaturbate.py @@ -15,11 +15,16 @@ def getVideoUrl(self): return self.getWantedResolutionPlaylist(self.lastInfo['url']) def getStatus(self): - headers = {"X-Requested-With": "XMLHttpRequest"} + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/53' + '7.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36', + "X-Requested-With": "XMLHttpRequest" + } data = {"room_slug": self.username, "bandwidth": "high"} try: - r = requests.post("https://chaturbate.com/get_edge_hls_url_ajax/", headers=headers, data=data) + r = requests.post("https://chaturbate.com/get_edge_hls_url_ajax/", + headers=headers, data=data) self.lastInfo = r.json() if self.lastInfo["room_status"] == "public": diff --git a/streamonitor/sites/stripchat.py b/streamonitor/sites/stripchat.py index 55870da..e26298a 100644 --- a/streamonitor/sites/stripchat.py +++ b/streamonitor/sites/stripchat.py @@ -31,20 +31,23 @@ def formatUrl(master, auto): return variants def getStatus(self): - r = requests.get('https://stripchat.com/api/vr/v2/models/username/' + self.username, headers=self.headers) - if r.status_code != 200: + try: + r = requests.get('https://stripchat.com/api/vr/v2/models/username/' + self.username, headers=self.headers) + if r.status_code != 200: + return Bot.Status.UNKNOWN + + self.lastInfo = r.json() + + if self.lastInfo["model"]["status"] == "public" and self.lastInfo["isCamAvailable"] and self.lastInfo['cam']["isCamActive"]: + return Bot.Status.PUBLIC + if self.lastInfo["model"]["status"] in ["private", "groupShow", "p2p", "virtualPrivate", "p2pVoice"]: + return Bot.Status.PRIVATE + if self.lastInfo["model"]["status"] in ["off", "idle"]: + return Bot.Status.OFFLINE + self.logger.warn(f'Got unknown status: {self.lastInfo["model"]["status"]}') + return Bot.Status.UNKNOWN + except: return Bot.Status.UNKNOWN - - self.lastInfo = r.json() - - if self.lastInfo["model"]["status"] == "public" and self.lastInfo["isCamAvailable"] and self.lastInfo['cam']["isCamActive"]: - return Bot.Status.PUBLIC - if self.lastInfo["model"]["status"] in ["private", "groupShow", "p2p", "virtualPrivate", "p2pVoice"]: - return Bot.Status.PRIVATE - if self.lastInfo["model"]["status"] in ["off", "idle"]: - return Bot.Status.OFFLINE - self.logger.warn(f'Got unknown status: {self.lastInfo["model"]["status"]}') - return Bot.Status.UNKNOWN Bot.loaded_sites.add(StripChat) diff --git a/test.py b/test.py new file mode 100644 index 0000000..ef21334 --- /dev/null +++ b/test.py @@ -0,0 +1,4 @@ +from streamonitor.managers import dashmanager + +if __name__ == "__main__": + dashmanager.HTTPManager().run()