From 4a252a514451e2037fcefdf7ef93c6a7e1c084d5 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 16 May 2023 00:37:08 +0300 Subject: [PATCH 01/25] Minor changes. --- .gitignore | 3 ++- parameters.py | 6 +++--- src/__init__.py | 0 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 src/__init__.py diff --git a/.gitignore b/.gitignore index e35b753..93cc4e3 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,5 @@ ENV/ # CUSTOM .idea/ config.json - +videos/* +downloads/* diff --git a/parameters.py b/parameters.py index 6dadd4c..ee7e3a5 100644 --- a/parameters.py +++ b/parameters.py @@ -1,13 +1,13 @@ DOWNLOADS_DIR = 'downloads' -MIN_FREE_DISK_PERCENT = 1.0 # in % +MIN_FREE_DISK_PERCENT = 30.0 # in % DEBUG = False # You can enter a number to select a specific height. # Use a huge number here and closest match to get the highest resolution variant # Eg: 240, 360, 480, 720, 1080, 1440, 99999 -WANTED_RESOLUTION = 1080 +WANTED_RESOLUTION = 3840 # Specify match type when specified height # Possible values: exact, exact_or_least_higher, exact_or_highest_lower, closest # Beware of the exact policy. Nothing gets downloaded if the wanted resolution is not available -WANTED_RESOLUTION_PREFERENCE = 'closest' \ No newline at end of file +WANTED_RESOLUTION_PREFERENCE = 'closest' diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 From 8a932881724b0ea3422f27c505e0236397acc864 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 16 May 2023 00:37:47 +0300 Subject: [PATCH 02/25] Add chopper script. Chooper cuts videos into smaller parts (f.e. 1 Gb). --- chopper.py | 74 +++++++++++++++ chopper_config.json | 4 + delete_less.py | 69 ++++++++++++++ src/ffmpeg-split.py | 225 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 372 insertions(+) create mode 100644 chopper.py create mode 100644 chopper_config.json create mode 100644 delete_less.py create mode 100644 src/ffmpeg-split.py diff --git a/chopper.py b/chopper.py new file mode 100644 index 0000000..895d260 --- /dev/null +++ b/chopper.py @@ -0,0 +1,74 @@ +import os +import sys +import threading +import time +import subprocess +import json + + +json_file = open('chopper_config.json', 'r') +json_data = json.load(json_file) +basepath = json_data['directory_path'] +size = json_data['max_size'] + +i = 0 +orders = [] +to_delete = [] + +black = '\x1b[30m' +red = '\x1b[31m' +green = '\x1b[32m' +yellow = '\x1b[33m' +blue = '\x1b[34m' +magenta = '\x1b[35m' +cyan = '\x1b[36m' +white = '\x1b[37m' +background_black = '\x1b[40m' +background_red = '\x1b[41m' +background_green = '\x1b[42m' +background_yellow = '\x1b[43m' +background_blue = '\x1b[44m' +background_magenta = '\x1b[45m' +background_cyan = '\x1b[46m' +background_white = '\x1b[47m' +reset = '\x1b[0m' + +# get full path +basepath = os.path.abspath(basepath) + +for entry in os.listdir(basepath): + if entry.endswith('.mp4'): + if not entry.startswith('.') and \ + os.path.isfile(os.path.join(basepath, entry)) and \ + os.path.getsize(os.path.join(basepath, entry)) > int(size*1.1): + i += 1 + orders.append( + f'python ./src/ffmpeg-split.py -f \"{os.path.join(basepath, entry)}\" -S {size}') + to_delete.append(os.path.join(basepath, entry)) + + +def run_command(command): + subprocess.call(command, shell=True) + + +for order in orders: + t = threading.Thread(target=run_command, args=(order,)) + t.start() + time.sleep(1) + +# if all threads are done, then exit +while True: + if len(threading.enumerate()) == 1: + break + else: + time.sleep(1) + +# delete original files +if len(to_delete) != 0: + print(f"chopper :: {yellow}[WARNING]{reset} Deleting original files...") + time.sleep(10) + for item in to_delete: + os.remove(item) + +print(f"chopper :: {green}[SUCCESS]{reset} Done!") +os.system("pause") diff --git a/chopper_config.json b/chopper_config.json new file mode 100644 index 0000000..688f505 --- /dev/null +++ b/chopper_config.json @@ -0,0 +1,4 @@ +{ + "directory_path": "videos", + "max_size": 1234234236 +} \ No newline at end of file diff --git a/delete_less.py b/delete_less.py new file mode 100644 index 0000000..73028aa --- /dev/null +++ b/delete_less.py @@ -0,0 +1,69 @@ +import sys +import argparse as arg +import os + + +black = '\x1b[30m' +red = '\x1b[31m' +green = '\x1b[32m' +yellow = '\x1b[33m' +blue = '\x1b[34m' +magenta = '\x1b[35m' +cyan = '\x1b[36m' +white = '\x1b[37m' +background_black = '\x1b[40m' +background_red = '\x1b[41m' +background_green = '\x1b[42m' +background_yellow = '\x1b[43m' +background_blue = '\x1b[44m' +background_magenta = '\x1b[45m' +background_cyan = '\x1b[46m' +background_white = '\x1b[47m' +reset = '\x1b[0m' + + +def delete_less(directory_path, size=200000000, ext="mp4"): + ext = "." + ext + counter = 0 + # Get the list of files in the directory + files = os.listdir(directory_path) + # get absolute path + directory_path = os.path.abspath(directory_path) + # Iterate over all the files + for file in files: + # Get the path of the file + file_path = os.path.join(directory_path, file) + # only mp4 + if file_path.endswith(ext): + # Get the size of the file in bytes + size_ = os.path.getsize(file_path) + # If size is less than 200 MB + if size_ < size: + # Delete the file + os.remove(file_path) + print(f"Deleted {file_path}") + counter += 1 + if counter != 0: + print( + f"delete_less :: {green}[SUCCESS]{reset} Deleted {counter} files less than {size} bytes.") + +# python delete_less200mb.py -p /home/username/Videos -s 200000000 + + +if __name__ == "__main__": + parser = arg.ArgumentParser(description="Delete files less than 200 MB") + parser.add_argument("-p", "--path", help="Path of the directory") + parser.add_argument("-s", "--size", help="Size of the file") + parser.add_argument("-e", "--ext", help="Extension of the file") + args = parser.parse_args() + # can be only path or path and size or path and size and ext + if args.path and args.size and args.ext: + delete_less(args.path, args.size, args.ext) + elif args.path and args.size: + delete_less(args.path, args.size) + elif args.path: + delete_less(args.path) + else: + print(f"{red}Please provide path of the directory{reset}") + sys.exit(1) + print(f'delete_less :: {green}[SUCCESS]{reset} Done!') diff --git a/src/ffmpeg-split.py b/src/ffmpeg-split.py new file mode 100644 index 0000000..09d60a2 --- /dev/null +++ b/src/ffmpeg-split.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import csv +import json +import math +import os +import shlex +import subprocess +from optparse import OptionParser + + +def split_by_manifest(filename, manifest, vcodec="copy", acodec="copy", + extra="", **kwargs): + """ Split video into segments based on the given manifest file. + + Arguments: + filename (str) - Location of the video. + manifest (str) - Location of the manifest file. + vcodec (str) - Controls the video codec for the ffmpeg video + output. + acodec (str) - Controls the audio codec for the ffmpeg video + output. + extra (str) - Extra options for ffmpeg. + """ + if not os.path.exists(manifest): + print("File does not exist: %s" % manifest) + raise SystemExit + + with open(manifest) as manifest_file: + manifest_type = manifest.split(".")[-1] + if manifest_type == "json": + config = json.load(manifest_file) + elif manifest_type == "csv": + config = csv.DictReader(manifest_file) + else: + print("Format not supported. File must be a csv or json file") + raise SystemExit + + split_cmd = ["ffmpeg", "-i", filename, "-vcodec", vcodec, + "-acodec", acodec, "-y"] + shlex.split(extra) + try: + fileext = filename.split(".")[-1] + except IndexError as e: + raise IndexError("No . in filename. Error: " + str(e)) + for video_config in config: + split_args = [] + try: + split_start = video_config["start_time"] + split_length = video_config.get("end_time", None) + if not split_length: + split_length = video_config["length"] + filebase = video_config["rename_to"] + if fileext in filebase: + filebase = ".".join(filebase.split(".")[:-1]) + + split_args += ["-ss", str(split_start), "-t", + str(split_length), filebase + "." + fileext] + print("########################################################") + print("About to run: " + " ".join(split_cmd + split_args)) + print("########################################################") + subprocess.check_output(split_cmd + split_args) + except KeyError as e: + print("############# Incorrect format ##############") + if manifest_type == "json": + print("The format of each json array should be:") + print("{start_time: , length: , rename_to: }") + elif manifest_type == "csv": + print("start_time,length,rename_to should be the first line ") + print("in the csv file.") + print("#############################################") + print(e) + raise SystemExit + + +def get_video_length(filename): + output = subprocess.check_output(("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", + "default=noprint_wrappers=1:nokey=1", filename)).strip() + video_length = int(float(output)) + print("Video length in seconds: " + str(video_length)) + + return video_length + + +def ceildiv(a, b): + return int(math.ceil(a / float(b))) + + +def split_by_seconds(filename, split_length, vcodec="copy", acodec="copy", + extra="", video_length=None, **kwargs): + if split_length and split_length <= 0: + print("Split length can't be 0") + raise SystemExit + + if not video_length: + video_length = get_video_length(filename) + split_count = ceildiv(video_length, split_length) + if split_count == 1: + print("Video length is less then the target split length.") + raise SystemExit + + split_cmd = ["ffmpeg", "-i", filename, "-vcodec", vcodec, "-acodec", acodec] + shlex.split(extra) + try: + filebase = ".".join(filename.split(".")[:-1]) + fileext = filename.split(".")[-1] + except IndexError as e: + raise IndexError("No . in filename. Error: " + str(e)) + for n in range(0, split_count): + split_args = [] + if n == 0: + split_start = 0 + else: + split_start = split_length * n + + split_args += ["-ss", str(split_start), "-t", str(split_length), + filebase + "-" + str(n + 1) + "-of-" + + str(split_count) + "." + fileext] + print("About to run: " + " ".join(split_cmd + split_args)) + subprocess.check_output(split_cmd + split_args) + + +def main(): + parser = OptionParser() + + parser.add_option("-f", "--file", + dest="filename", + help="File to split, for example sample.avi", + type="string", + action="store" + ) + parser.add_option("-s", "--split-size", + dest="split_length", + help="Split or chunk size in seconds, for example 10", + type="int", + action="store" + ) + parser.add_option("-c", "--split-chunks", + dest="split_chunks", + help="Number of chunks to split to", + type="int", + action="store" + ) + parser.add_option("-S", "--split-filesize", + dest="split_filesize", + help="Split or chunk size in bytes (approximate)", + type="int", + action="store" + ) + parser.add_option("--filesize-factor", + dest="filesize_factor", + help="with --split-filesize, use this factor in time to" + " size heuristics [default: %default]", + type="float", + action="store", + default=0.95 + ) + parser.add_option("--chunk-strategy", + dest="chunk_strategy", + help="with --split-filesize, allocate chunks according to" + " given strategy (eager or even)", + type="choice", + action="store", + choices=['eager', 'even'], + default='eager' + ) + parser.add_option("-m", "--manifest", + dest="manifest", + help="Split video based on a json manifest file. ", + type="string", + action="store" + ) + parser.add_option("-v", "--vcodec", + dest="vcodec", + help="Video codec to use. ", + type="string", + default="copy", + action="store" + ) + parser.add_option("-a", "--acodec", + dest="acodec", + help="Audio codec to use. ", + type="string", + default="copy", + action="store" + ) + parser.add_option("-e", "--extra", + dest="extra", + help="Extra options for ffmpeg, e.g. '-e -threads 8'. ", + type="string", + default="", + action="store" + ) + (options, args) = parser.parse_args() + + def bailout(): + parser.print_help() + raise SystemExit + + if not options.filename: + bailout() + + if options.manifest: + split_by_manifest(**options.__dict__) + else: + video_length = None + if not options.split_length: + video_length = get_video_length(options.filename) + file_size = os.stat(options.filename).st_size + split_filesize = None + if options.split_filesize: + split_filesize = int(options.split_filesize * options.filesize_factor) + if split_filesize and options.chunk_strategy == 'even': + options.split_chunks = ceildiv(file_size, split_filesize) + if options.split_chunks: + options.split_length = ceildiv(video_length, options.split_chunks) + if not options.split_length and split_filesize: + options.split_length = int(split_filesize / float(file_size) * video_length) + if not options.split_length: + bailout() + split_by_seconds(video_length=video_length, **options.__dict__) + + +if __name__ == '__main__': + main() From eaa663102c9bb47a59b54adec2d1d40725e89633 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 16 May 2023 00:39:26 +0300 Subject: [PATCH 03/25] Add automating script. This script can pull vidoes from all folders into 'videos' folder and cut them into smaller parts, using chopper. --- move_all_captures.bat | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 move_all_captures.bat diff --git a/move_all_captures.bat b/move_all_captures.bat new file mode 100644 index 0000000..3e3739b --- /dev/null +++ b/move_all_captures.bat @@ -0,0 +1,38 @@ +@ECHO OFF +TITLE WAIT ! +:: ASSIGN THE FILE PATH OF BATCH FILE TO A VARIABLE +SET "sourceDir=%CD%\downloads" +:: GET THE NAME OF THE FOLDER WHICH THE BATCH FILE IS IN +FOR %%a IN (.) DO SET currentFolder=%%~na +:: GO UP ONE DIRECTORY +::CD .. +:: MAKE A DYNAMIC FOLDER NAME +::SET folderName=Copied From %currentFolder% +SET "folderName=videos" +:: CREATE A FOLDER TO PUT THE COPIED FILES IN +:: IF FOLDER ALREADY EXISTS DELETE IT +IF NOT EXIST "%folderName%" MKDIR "%folderName%" +:: ASSIGN DESTINATION FOLDER TO A VARIABLE +SET "destinationFolder=%CD%\%folderName%" +:: CREATE A LOG FILE IN DESTINATION FOLDER +SET "_report=%destinationFolder%\logxcopy.txt" +:: CREATE ERROR MESSAGE +IF NOT EXIST "%sourceDir%" (ECHO.Could not find %sourceDir% &GoTo:DONE) +:: OVERWRITE PREVIOUS LOG +>"%_report%" ( +echo.%date% – %time% +echo.————————————————— +echo. +) +:: COPY FILES +FOR /F "Delims=" %%! IN ('DIR "%sourceDir%\" /b /s /a-d 2^>NUL') DO ( +@ECHO.%%! &( +@MOVE /Y "%%!" "%destinationFolder%\") +) +python .\delete_less.py -p "./videos/" +timeout 5 > NUL +python .\chopper.py +:DONE +TITLE,Done... +echo Done! +ECHO.&PAUSE>NUL \ No newline at end of file From 7f7c840b3b6ef171fe53ff77218b4736fd1e77ef Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Wed, 31 May 2023 07:25:41 +0300 Subject: [PATCH 04/25] Code refactoring --- .vscode/settings.json | 9 + chopper.py | 74 ---- chopper_config.json | 4 - config.json | 468 ++++++++++++++++++++++- mac.bat | 1 + mac.py | 66 ++++ move_all_captures.bat | 38 -- parameters.py | 2 +- requirements.txt | 4 +- run.bat | 1 + delete_less.py => src/delete_less.py | 45 +-- src/{ffmpeg-split.py => ffmpeg_split.py} | 41 +- src/move_all.py | 82 ++++ src/splitter.py | 100 +++++ 14 files changed, 777 insertions(+), 158 deletions(-) create mode 100644 .vscode/settings.json delete mode 100644 chopper.py delete mode 100644 chopper_config.json create mode 100644 mac.bat create mode 100644 mac.py delete mode 100644 move_all_captures.bat create mode 100644 run.bat rename delete_less.py => src/delete_less.py (66%) rename src/{ffmpeg-split.py => ffmpeg_split.py} (85%) create mode 100644 src/move_all.py create mode 100644 src/splitter.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d4379c1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + }, + "python.formatting.provider": "none", + "githubPullRequests.ignoredPullRequestBranches": [ + "master" + ] +} \ No newline at end of file diff --git a/chopper.py b/chopper.py deleted file mode 100644 index 895d260..0000000 --- a/chopper.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import sys -import threading -import time -import subprocess -import json - - -json_file = open('chopper_config.json', 'r') -json_data = json.load(json_file) -basepath = json_data['directory_path'] -size = json_data['max_size'] - -i = 0 -orders = [] -to_delete = [] - -black = '\x1b[30m' -red = '\x1b[31m' -green = '\x1b[32m' -yellow = '\x1b[33m' -blue = '\x1b[34m' -magenta = '\x1b[35m' -cyan = '\x1b[36m' -white = '\x1b[37m' -background_black = '\x1b[40m' -background_red = '\x1b[41m' -background_green = '\x1b[42m' -background_yellow = '\x1b[43m' -background_blue = '\x1b[44m' -background_magenta = '\x1b[45m' -background_cyan = '\x1b[46m' -background_white = '\x1b[47m' -reset = '\x1b[0m' - -# get full path -basepath = os.path.abspath(basepath) - -for entry in os.listdir(basepath): - if entry.endswith('.mp4'): - if not entry.startswith('.') and \ - os.path.isfile(os.path.join(basepath, entry)) and \ - os.path.getsize(os.path.join(basepath, entry)) > int(size*1.1): - i += 1 - orders.append( - f'python ./src/ffmpeg-split.py -f \"{os.path.join(basepath, entry)}\" -S {size}') - to_delete.append(os.path.join(basepath, entry)) - - -def run_command(command): - subprocess.call(command, shell=True) - - -for order in orders: - t = threading.Thread(target=run_command, args=(order,)) - t.start() - time.sleep(1) - -# if all threads are done, then exit -while True: - if len(threading.enumerate()) == 1: - break - else: - time.sleep(1) - -# delete original files -if len(to_delete) != 0: - print(f"chopper :: {yellow}[WARNING]{reset} Deleting original files...") - time.sleep(10) - for item in to_delete: - os.remove(item) - -print(f"chopper :: {green}[SUCCESS]{reset} Done!") -os.system("pause") diff --git a/chopper_config.json b/chopper_config.json deleted file mode 100644 index 688f505..0000000 --- a/chopper_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "directory_path": "videos", - "max_size": 1234234236 -} \ No newline at end of file diff --git a/config.json b/config.json index 0637a08..61cd9dd 100644 --- a/config.json +++ b/config.json @@ -1 +1,467 @@ -[] \ No newline at end of file +[ + { + "site": "StripChat", + "username": "raingurl", + "running": true + }, + { + "site": "StripChat", + "username": "Kralya_Smile", + "running": true + }, + { + "site": "StripChat", + "username": "lika_lil", + "running": true + }, + { + "site": "StripChat", + "username": "Misss_Vikk", + "running": true + }, + { + "site": "StripChat", + "username": "Scops_owl", + "running": true + }, + { + "site": "StripChat", + "username": "boobsybootie", + "running": true + }, + { + "site": "StripChat", + "username": "kassablanca_", + "running": true + }, + { + "site": "StripChat", + "username": "littlecarolina", + "running": true + }, + { + "site": "StripChat", + "username": "moon_shy", + "running": true + }, + { + "site": "StripChat", + "username": "Careful_i_bite", + "running": true + }, + { + "site": "StripChat", + "username": "Ginger_Pie", + "running": true + }, + { + "site": "StripChat", + "username": "BUNY_PRINCESS", + "running": true + }, + { + "site": "StripChat", + "username": "kate_pierce", + "running": true + }, + { + "site": "StripChat", + "username": "_Lory_Like_", + "running": true + }, + { + "site": "StripChat", + "username": "Filla--", + "running": true + }, + { + "site": "StripChat", + "username": "susu-dy", + "running": true + }, + { + "site": "StripChat", + "username": "-meowpa-", + "running": true + }, + { + "site": "StripChat", + "username": "MaryBrent", + "running": true + }, + { + "site": "StripChat", + "username": "EroticNana", + "running": true + }, + { + "site": "StripChat", + "username": "FiOnee", + "running": true + }, + { + "site": "StripChat", + "username": "Kitty_Killer_", + "running": true + }, + { + "site": "StripChat", + "username": "RoseMarceau_", + "running": true + }, + { + "site": "StripChat", + "username": "Lana_Lee_", + "running": true + }, + { + "site": "StripChat", + "username": "Rocks_Babies_", + "running": true + }, + { + "site": "StripChat", + "username": "Kassy_Sun", + "running": true + }, + { + "site": "StripChat", + "username": "Vilgelminaa", + "running": true + }, + { + "site": "StripChat", + "username": "ChloeAttwood_", + "running": true + }, + { + "site": "StripChat", + "username": "DaddysLittleSlutX", + "running": true + }, + { + "site": "StripChat", + "username": "doll_joy", + "running": true + }, + { + "site": "StripChat", + "username": "AkiShina", + "running": true + }, + { + "site": "StripChat", + "username": "Tsumugi_M", + "running": true + }, + { + "site": "StripChat", + "username": "JessAdams", + "running": true + }, + { + "site": "StripChat", + "username": "YourMisano", + "running": true + }, + { + "site": "StripChat", + "username": "TayaMeow", + "running": true + }, + { + "site": "StripChat", + "username": "Eva_Bear", + "running": true + }, + { + "site": "StripChat", + "username": "Badassloli_", + "running": true + }, + { + "site": "StripChat", + "username": "cutiesuee", + "running": true + }, + { + "site": "StripChat", + "username": "amaya_mori", + "running": true + }, + { + "site": "StripChat", + "username": "aerri_lee", + "running": true + }, + { + "site": "StripChat", + "username": "asuna_love", + "running": true + }, + { + "site": "StripChat", + "username": "emma_moanss", + "running": true + }, + { + "site": "BongaCams", + "username": "TammyLynn", + "running": true + }, + { + "site": "BongaCams", + "username": "HollyTailor", + "running": true + }, + { + "site": "StripChat", + "username": "Runa_star", + "running": true + }, + { + "site": "BongaCams", + "username": "Pussy-", + "running": true + }, + { + "site": "BongaCams", + "username": "MiaTonyy", + "running": true + }, + { + "site": "BongaCams", + "username": "PlushCatt", + "running": true + }, + { + "site": "BongaCams", + "username": "milky-cunt", + "running": true + }, + { + "site": "BongaCams", + "username": "WhiteGirl-one", + "running": true + }, + { + "site": "BongaCams", + "username": "Bestblondie", + "running": true + }, + { + "site": "BongaCams", + "username": "-wellcum-", + "running": true + }, + { + "site": "BongaCams", + "username": "MaxineDiaz", + "running": true + }, + { + "site": "BongaCams", + "username": "Bestblondie", + "running": true + }, + { + "site": "Chaturbate", + "username": "donnaayanami", + "running": true + }, + { + "site": "Chaturbate", + "username": "vixenp", + "running": true + }, + { + "site": "Chaturbate", + "username": "alice_kosmos", + "running": true + }, + { + "site": "Chaturbate", + "username": "beka_hernandez", + "running": true + }, + { + "site": "Chaturbate", + "username": "eva_elfyy", + "running": true + }, + { + "site": "Chaturbate", + "username": "francesdonna", + "running": true + }, + { + "site": "Chaturbate", + "username": "aura_69", + "running": true + }, + { + "site": "Chaturbate", + "username": "marcussanchez", + "running": true + }, + { + "site": "BongaCams", + "username": "Alisha008", + "running": true + }, + { + "site": "BongaCams", + "username": "Kittenmeew", + "running": true + }, + { + "site": "BongaCams", + "username": "AbsintheGirl", + "running": true + }, + { + "site": "BongaCams", + "username": "MeyShan", + "running": true + }, + { + "site": "BongaCams", + "username": "NeNaSuTnA", + "running": true + }, + { + "site": "BongaCams", + "username": "MarinnaCat", + "running": true + }, + { + "site": "BongaCams", + "username": "KiraLiLime", + "running": true + }, + { + "site": "StripChat", + "username": "SamanthaMorgann", + "running": true + }, + { + "site": "StripChat", + "username": "MadissonSoft", + "running": true + }, + { + "site": "StripChat", + "username": "ummpaLummpa", + "running": true + }, + { + "site": "StripChat", + "username": "wendy_taylor_", + "running": true + }, + { + "site": "BongaCams", + "username": "barsaa7", + "running": true + }, + { + "site": "BongaCams", + "username": "Arisha-w", + "running": true + }, + { + "site": "BongaCams", + "username": "Effy_S", + "running": true + }, + { + "site": "BongaCams", + "username": "SoiaPenka", + "running": true + }, + { + "site": "BongaCams", + "username": "elizabethglor", + "running": true + }, + { + "site": "BongaCams", + "username": "Kralya-Smile", + "running": true + }, + { + "site": "BongaCams", + "username": "LilyaLilac", + "running": true + }, + { + "site": "BongaCams", + "username": "cyberannita", + "running": true + }, + { + "site": "BongaCams", + "username": "Baby-Tisha", + "running": true + }, + { + "site": "BongaCams", + "username": "Annelitt", + "running": true + }, + { + "site": "BongaCams", + "username": "AmeliaLean", + "running": true + }, + { + "site": "BongaCams", + "username": "syndicete", + "running": true + }, + { + "site": "BongaCams", + "username": "MiaandMelissa", + "running": true + }, + { + "site": "BongaCams", + "username": "EvaSSly", + "running": true + }, + { + "site": "BongaCams", + "username": "Aru-Koto", + "running": true + }, + { + "site": "BongaCams", + "username": "MariawwEmma", + "running": true + }, + { + "site": "BongaCams", + "username": "LISA-ALYSSA", + "running": true + }, + { + "site": "BongaCams", + "username": "JennaStroker", + "running": true + }, + { + "site": "BongaCams", + "username": "FreyaAdams", + "running": true + }, + { + "site": "BongaCams", + "username": "Lilit1201", + "running": true + }, + { + "site": "Chaturbate", + "username": "fiassquad", + "running": true + } +] \ No newline at end of file diff --git a/mac.bat b/mac.bat new file mode 100644 index 0000000..7cf84ee --- /dev/null +++ b/mac.bat @@ -0,0 +1 @@ +python mac.py \ No newline at end of file diff --git a/mac.py b/mac.py new file mode 100644 index 0000000..77fd10a --- /dev/null +++ b/mac.py @@ -0,0 +1,66 @@ +import os +import sys +import colorama as col +import threading as th +from tqdm import tqdm + +from src import move_all as ma +from src import splitter as sp +from src import delete_less as dl + + +def move_and_split(src: str, dst: str, file_size: int = None, log: bool = True, + tqdm_opt: bool = True): + """ + Move all files from subdirectories of src into dst. + Then split all video files in dst into file_size chunks + and delete all original files. + + Parameters + ---------- + src (str): source directory + dst (str): destination directory + file_size (int): size of chunks in Bytes, default is None + """ + ma.move_all_subs(src, dst, log=True) + ma.delete_empty_dirs(src, log=True) + # find all video files in dst + files = [] + for root, dirs, filenames in os.walk(dst): + for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\ +[MAC]: Finding video files')\ + if tqdm_opt else filenames: + if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')): + files.append(os.path.join(root, filename)) + # split all video files in dst + + def call_function(filename): + try: + sp.main_split(filename, split_filesize=file_size, tqdm_opt=False) + except Exception as e: + print( + f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} {e}') + except KeyboardInterrupt: + print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} Keyboard interrupt') + + threads = [] + for f in tqdm(files, desc=f'{col.Style.RESET_ALL}[MAC]: Starting threads')\ + if tqdm_opt else files: + t = th.Thread(target=call_function, args=(f,)) + threads.append(t) + t.start() + + for t in tqdm(threads, desc=f'{col.Style.RESET_ALL}[MAC]: Splitting videos')\ + if tqdm_opt else threads: + t.join() + if log: + print(f'{col.Style.RESET_ALL}[MAC]: {col.Fore.GREEN}\ +DONE!{col.Style.RESET_ALL}') + + +if __name__ == '__main__': + move_and_split('downloads', 'videos', file_size=2000000000) + dl.delete_less('videos', 200000000, log=True) + input() diff --git a/move_all_captures.bat b/move_all_captures.bat deleted file mode 100644 index 3e3739b..0000000 --- a/move_all_captures.bat +++ /dev/null @@ -1,38 +0,0 @@ -@ECHO OFF -TITLE WAIT ! -:: ASSIGN THE FILE PATH OF BATCH FILE TO A VARIABLE -SET "sourceDir=%CD%\downloads" -:: GET THE NAME OF THE FOLDER WHICH THE BATCH FILE IS IN -FOR %%a IN (.) DO SET currentFolder=%%~na -:: GO UP ONE DIRECTORY -::CD .. -:: MAKE A DYNAMIC FOLDER NAME -::SET folderName=Copied From %currentFolder% -SET "folderName=videos" -:: CREATE A FOLDER TO PUT THE COPIED FILES IN -:: IF FOLDER ALREADY EXISTS DELETE IT -IF NOT EXIST "%folderName%" MKDIR "%folderName%" -:: ASSIGN DESTINATION FOLDER TO A VARIABLE -SET "destinationFolder=%CD%\%folderName%" -:: CREATE A LOG FILE IN DESTINATION FOLDER -SET "_report=%destinationFolder%\logxcopy.txt" -:: CREATE ERROR MESSAGE -IF NOT EXIST "%sourceDir%" (ECHO.Could not find %sourceDir% &GoTo:DONE) -:: OVERWRITE PREVIOUS LOG ->"%_report%" ( -echo.%date% – %time% -echo.————————————————— -echo. -) -:: COPY FILES -FOR /F "Delims=" %%! IN ('DIR "%sourceDir%\" /b /s /a-d 2^>NUL') DO ( -@ECHO.%%! &( -@MOVE /Y "%%!" "%destinationFolder%\") -) -python .\delete_less.py -p "./videos/" -timeout 5 > NUL -python .\chopper.py -:DONE -TITLE,Done... -echo Done! -ECHO.&PAUSE>NUL \ No newline at end of file diff --git a/parameters.py b/parameters.py index ee7e3a5..843dbd9 100644 --- a/parameters.py +++ b/parameters.py @@ -1,5 +1,5 @@ DOWNLOADS_DIR = 'downloads' -MIN_FREE_DISK_PERCENT = 30.0 # in % +MIN_FREE_DISK_PERCENT = 60.0 # in % DEBUG = False # You can enter a number to select a specific height. diff --git a/requirements.txt b/requirements.txt index f688ab5..d07a493 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,6 @@ termcolor beautifulsoup4 websocket-client ffmpy -m3u8 \ No newline at end of file +m3u8 +colorama +tqdm \ No newline at end of file diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..5077a0d --- /dev/null +++ b/run.bat @@ -0,0 +1 @@ +python Downloader.py \ No newline at end of file diff --git a/delete_less.py b/src/delete_less.py similarity index 66% rename from delete_less.py rename to src/delete_less.py index 73028aa..ace43a2 100644 --- a/delete_less.py +++ b/src/delete_less.py @@ -1,29 +1,21 @@ import sys import argparse as arg import os +import colorama as col -black = '\x1b[30m' -red = '\x1b[31m' -green = '\x1b[32m' -yellow = '\x1b[33m' -blue = '\x1b[34m' -magenta = '\x1b[35m' -cyan = '\x1b[36m' -white = '\x1b[37m' -background_black = '\x1b[40m' -background_red = '\x1b[41m' -background_green = '\x1b[42m' -background_yellow = '\x1b[43m' -background_blue = '\x1b[44m' -background_magenta = '\x1b[45m' -background_cyan = '\x1b[46m' -background_white = '\x1b[47m' -reset = '\x1b[0m' +def delete_less(directory_path: str, size: int = 200000000, ext: str = ".mp4", + log: bool = True): + """ + Delete all files in directory_path that are less than size in bytes. - -def delete_less(directory_path, size=200000000, ext="mp4"): - ext = "." + ext + Parameters + ---------- + directory_path (str): path to directory + size (int): size in bytes, default is 200000000 + ext (str): extension of files to delete, default is ".mp4" + log (bool): if True, print all deleted files + """ counter = 0 # Get the list of files in the directory files = os.listdir(directory_path) @@ -41,13 +33,14 @@ def delete_less(directory_path, size=200000000, ext="mp4"): if size_ < size: # Delete the file os.remove(file_path) - print(f"Deleted {file_path}") + if log: + print( + f'{col.Style.RESET_ALL}[DELETER]: \"{file_path}\" \ +deleted') counter += 1 - if counter != 0: - print( - f"delete_less :: {green}[SUCCESS]{reset} Deleted {counter} files less than {size} bytes.") - -# python delete_less200mb.py -p /home/username/Videos -s 200000000 + if log: + print(f'{col.Style.RESET_ALL}[DELETER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') if __name__ == "__main__": diff --git a/src/ffmpeg-split.py b/src/ffmpeg_split.py similarity index 85% rename from src/ffmpeg-split.py rename to src/ffmpeg_split.py index 09d60a2..206dcec 100644 --- a/src/ffmpeg-split.py +++ b/src/ffmpeg_split.py @@ -9,6 +9,7 @@ import shlex import subprocess from optparse import OptionParser +from tqdm import tqdm def split_by_manifest(filename, manifest, vcodec="copy", acodec="copy", @@ -75,11 +76,13 @@ def split_by_manifest(filename, manifest, vcodec="copy", acodec="copy", def get_video_length(filename): - output = subprocess.check_output(("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", - "default=noprint_wrappers=1:nokey=1", filename)).strip() - video_length = int(float(output)) - print("Video length in seconds: " + str(video_length)) - + try: + output = subprocess.check_output(("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", + "default=noprint_wrappers=1:nokey=1", filename)).strip() + video_length = int(float(output)) + # print("Video length in seconds: " + str(video_length)) + except Exception as e: + raise SystemExit return video_length @@ -87,8 +90,13 @@ def ceildiv(a, b): return int(math.ceil(a / float(b))) +def get_name_from_path(path): + return os.path.splitext(os.path.basename(path))[0] + + def split_by_seconds(filename, split_length, vcodec="copy", acodec="copy", - extra="", video_length=None, **kwargs): + extra="", video_length=None, tqdm_opt: bool = True, + **kwargs): if split_length and split_length <= 0: print("Split length can't be 0") raise SystemExit @@ -97,16 +105,20 @@ def split_by_seconds(filename, split_length, vcodec="copy", acodec="copy", video_length = get_video_length(filename) split_count = ceildiv(video_length, split_length) if split_count == 1: - print("Video length is less then the target split length.") + # print("Video length is less then the target split length.") raise SystemExit - split_cmd = ["ffmpeg", "-i", filename, "-vcodec", vcodec, "-acodec", acodec] + shlex.split(extra) + split_cmd = ["ffmpeg", "-i", filename, "-vcodec", + vcodec, "-acodec", acodec] + shlex.split(extra) try: filebase = ".".join(filename.split(".")[:-1]) fileext = filename.split(".")[-1] except IndexError as e: raise IndexError("No . in filename. Error: " + str(e)) - for n in range(0, split_count): + for n in tqdm( + range(0, split_count), + desc=f"Splitting {get_name_from_path(filename) + '.' + fileext}")\ + if tqdm_opt else range(0, split_count): split_args = [] if n == 0: split_start = 0 @@ -116,7 +128,7 @@ def split_by_seconds(filename, split_length, vcodec="copy", acodec="copy", split_args += ["-ss", str(split_start), "-t", str(split_length), filebase + "-" + str(n + 1) + "-of-" + str(split_count) + "." + fileext] - print("About to run: " + " ".join(split_cmd + split_args)) + # print("About to run: " + " ".join(split_cmd + split_args)) subprocess.check_output(split_cmd + split_args) @@ -209,13 +221,16 @@ def bailout(): file_size = os.stat(options.filename).st_size split_filesize = None if options.split_filesize: - split_filesize = int(options.split_filesize * options.filesize_factor) + split_filesize = int( + options.split_filesize * options.filesize_factor) if split_filesize and options.chunk_strategy == 'even': options.split_chunks = ceildiv(file_size, split_filesize) if options.split_chunks: - options.split_length = ceildiv(video_length, options.split_chunks) + options.split_length = ceildiv( + video_length, options.split_chunks) if not options.split_length and split_filesize: - options.split_length = int(split_filesize / float(file_size) * video_length) + options.split_length = int( + split_filesize / float(file_size) * video_length) if not options.split_length: bailout() split_by_seconds(video_length=video_length, **options.__dict__) diff --git a/src/move_all.py b/src/move_all.py new file mode 100644 index 0000000..6291a77 --- /dev/null +++ b/src/move_all.py @@ -0,0 +1,82 @@ +import os +import sys +import colorama as col + + +def move_all_subs(src: str, dst: str, ext: str = None, log: bool = False): + """ + Move all files from subdirectories of src into dst. + + Parameters + ---------- + src (str): source directory + dst (str): destination directory + ext (str): extension of files to move, default is None + log (bool): if True, print all moved files + """ + if not os.path.isdir(src): + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} \ +\"{src}\" is not a directory') + if not os.path.isdir(dst): + os.mkdir(dst) + + for root, dirs, files in os.walk(src): + for f in files: + if ext is None or f.endswith(ext): + src_path = os.path.join(root, f) + dst_path = os.path.join(dst, f) + try: + os.rename(src_path, dst_path) + if log: + print(f'{col.Style.RESET_ALL}[MOVER]: \"{src_path}\" \ +-> \"{dst_path}\"') + except FileExistsError: + print( + f'{col.Style.RESET_ALL}[MOVER]: \ +{col.Fore.RED}ERROR{col.Style.RESET_ALL}: \ +\"{dst_path}\" already exists') + continue + print(f'{col.Style.RESET_ALL}[MOVER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') + + +def delete_empty_dirs(src: str, log: bool = False): + """ + Delete all empty subdirectories of src. + + Parameters + ---------- + src (str): source directory + log (bool): if True, print all deleted directories + """ + if not os.path.isdir(src): + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} \ +\"{src}\" is not a directory') + sys.exit(1) + + for root, dirs, files in os.walk(src, topdown=False): + for d in dirs: + d_path = os.path.join(root, d) + if not os.listdir(d_path): + try: + os.rmdir(d_path) + if log: + print(f'{col.Style.RESET_ALL}[DELETER]: \"{d_path}\" \ +deleted') + except OSError: + print( + f'{col.Style.RESET_ALL}[DELETER]: {col.Fore.RED}\ +ERROR{col.Style.RESET_ALL}: \ +\"{d_path}\" is not empty') + continue + print(f'{col.Style.RESET_ALL}[DELETER]: \ +{col.Fore.GREEN}DONE!{col.Style.RESET_ALL}') + + +if __name__ == '__main__': + move_all_subs('downloads', 'videos', ext='.mp4', log=True) + delete_empty_dirs('downloads', log=True) diff --git a/src/splitter.py b/src/splitter.py new file mode 100644 index 0000000..892a50c --- /dev/null +++ b/src/splitter.py @@ -0,0 +1,100 @@ +import os +import sys +import colorama as col +from . import ffmpeg_split as ffs + + +def main_split(filename: str, tqdm_opt: bool = True, split_filesize: int = None, + filesize_factor: float = 0.95, chunk_strategy: str = 'eager', + split_chunks: int = None, split_length: int = None, + vcodec: str = 'copy', acodec: str = 'copy', + extra: str = '-hide_banner -loglevel quiet -y'): + """ + Split video based on a set of options. + + Parameters + ---------- + filename (str): Path to the video file. + tqdm_opt (bool): Show progress bar. Default is True. + split_filesize (int): Split video based on a target filesize in bytes. + filesize_factor (float): Factor to apply to the filesize. \ +Default is 1.0. + chunk_strategy (str): Strategy to use when splitting by filesize. \ +Default is 'even'. + split_chunks (int): Split video into a set number of chunks. + split_length (int): Split video into chunks of a set length in seconds. + vcodec (str): Video codec to use. Default is 'copy'. + acodec (str): Audio codec to use. Default is 'copy'. + extra (str): Extra options for ffmpeg, e.g. '-e -threads 8'. \ +Default is ''. + """ + video_length = ffs.get_video_length(filename) + file_size = os.stat(filename).st_size + split_filesize = int(split_filesize * filesize_factor) + if file_size < split_filesize * 1.1: + return + if split_filesize and chunk_strategy == 'even': + split_chunks = ffs.ceildiv(file_size, split_filesize) + if split_chunks != None: + split_length = ffs.ceildiv(video_length, split_chunks) + if not split_length and split_filesize: + split_length = int( + split_filesize / float(file_size) * video_length) + try: + ffs.split_by_seconds( + video_length=video_length, tqdm_opt=tqdm_opt, + **craft_options(filename, split_filesize, filesize_factor, + chunk_strategy, split_chunks, split_length, + vcodec, acodec, extra)) + os.remove(filename) + except KeyboardInterrupt: + print(col.Fore.RED + 'Aborted by user.' + col.Style.RESET_ALL) + sys.exit(1) + + +def craft_options(filename: str, split_filesize: int = None, + filesize_factor: float = 1.0, chunk_strategy: str = '', + split_chunks: int = None, split_length: int = None, + vcodec: str = 'copy', acodec: str = 'copy', extra: str = '', + manifest: str = None): + """ + Craft options for ffmpeg_split.split_by_seconds(). + + Parameters + ---------- + filename (str): Path to the video file. + split_filesize (int): Split video based on a target filesize in bytes. + filesize_factor (float): Factor to apply to the filesize. \ +Default is 1.0. + chunk_strategy (str): Strategy to use when splitting by filesize. \ +Default is 'even'. + split_chunks (int): Split video into a set number of chunks. + split_length (int): Split video into chunks of a set length in seconds. + vcodec (str): Video codec to use. Default is 'copy'. + acodec (str): Audio codec to use. Default is 'copy'. + extra (str): Extra options for ffmpeg, e.g. '-e -threads 8'. \ +Default is ''. + manifest (str): Split video based on a json manifest file. + + Returns + ------- + options (dict): Options for ffmpeg_split.split_by_seconds(). + """ + options = { + 'filename': filename, + 'split_filesize': split_filesize, + 'filesize_factor': filesize_factor, + 'chunk_strategy': chunk_strategy, + 'split_chunks': split_chunks, + 'split_length': split_length, + 'vcodec': vcodec, + 'acodec': acodec, + 'extra': extra, + 'manifest': manifest + } + return options + + +if __name__ == '__main__': + main_split('./videos/2023-05-22 08-14-39.mp4', + split_filesize=500000000) From 5e585576594369ee0da62ef93f8e8d5fdbb369da Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Fri, 2 Jun 2023 01:42:45 +0300 Subject: [PATCH 05/25] Chaturbate headers fix --- config.json | 102 +++++++++++++++++++++++++++++-- src/move_all.py | 14 +++-- src/splitter.py | 2 +- streamonitor/sites/chaturbate.py | 9 ++- streamonitor/sites/stripchat.py | 29 +++++---- 5 files changed, 130 insertions(+), 26 deletions(-) diff --git a/config.json b/config.json index 61cd9dd..cf5e2b5 100644 --- a/config.json +++ b/config.json @@ -204,11 +204,6 @@ "username": "emma_moanss", "running": true }, - { - "site": "BongaCams", - "username": "TammyLynn", - "running": true - }, { "site": "BongaCams", "username": "HollyTailor", @@ -381,7 +376,7 @@ }, { "site": "BongaCams", - "username": "elizabethglor", + "username": "ElizabethGlor", "running": true }, { @@ -463,5 +458,100 @@ "site": "Chaturbate", "username": "fiassquad", "running": true + }, + { + "site": "Chaturbate", + "username": "ann_lover016", + "running": true + }, + { + "site": "Chaturbate", + "username": "missnicolette1", + "running": true + }, + { + "site": "Chaturbate", + "username": "hornybunnys", + "running": true + }, + { + "site": "Chaturbate", + "username": "lollypetitte_", + "running": true + }, + { + "site": "Chaturbate", + "username": "fairyinthewild", + "running": true + }, + { + "site": "Chaturbate", + "username": "pornxxxcouple", + "running": true + }, + { + "site": "Chaturbate", + "username": "sweetpornface", + "running": true + }, + { + "site": "Chaturbate", + "username": "santiago_and_lana", + "running": true + }, + { + "site": "Chaturbate", + "username": "angelandfairieskink", + "running": true + }, + { + "site": "BongaCams", + "username": "Alastaira", + "running": true + }, + { + "site": "BongaCams", + "username": "Vasilisa-V", + "running": true + }, + { + "site": "BongaCams", + "username": "ElisBraun", + "running": true + }, + { + "site": "BongaCams", + "username": "Diafoxy", + "running": true + }, + { + "site": "BongaCams", + "username": "Lola3000", + "running": true + }, + { + "site": "BongaCams", + "username": "candykushhh", + "running": true + }, + { + "site": "BongaCams", + "username": "KateDanverr", + "running": true + }, + { + "site": "BongaCams", + "username": "crispy-tati-", + "running": true + }, + { + "site": "BongaCams", + "username": "MissCatty", + "running": true + }, + { + "site": "BongaCams", + "username": "HotDemonic", + "running": true } ] \ No newline at end of file diff --git a/src/move_all.py b/src/move_all.py index 6291a77..5b7be92 100644 --- a/src/move_all.py +++ b/src/move_all.py @@ -28,10 +28,16 @@ def move_all_subs(src: str, dst: str, ext: str = None, log: bool = False): src_path = os.path.join(root, f) dst_path = os.path.join(dst, f) try: - os.rename(src_path, dst_path) - if log: - print(f'{col.Style.RESET_ALL}[MOVER]: \"{src_path}\" \ --> \"{dst_path}\"') + # check if file occupied by another process + if os.access(src_path, os.W_OK): + os.rename(src_path, dst_path) + if log: + print(f'{col.Style.RESET_ALL}[MOVER]: \ +\"{src_path}\" -> \"{dst_path}\"') + except PermissionError: + print( + f'{col.Style.RESET_ALL}[MOVER]: {col.Fore.RED}\ +ERROR{col.Style.RESET_ALL}: \"{src_path}\" is occupied by another process') except FileExistsError: print( f'{col.Style.RESET_ALL}[MOVER]: \ diff --git a/src/splitter.py b/src/splitter.py index 892a50c..a151672 100644 --- a/src/splitter.py +++ b/src/splitter.py @@ -31,7 +31,7 @@ def main_split(filename: str, tqdm_opt: bool = True, split_filesize: int = None, video_length = ffs.get_video_length(filename) file_size = os.stat(filename).st_size split_filesize = int(split_filesize * filesize_factor) - if file_size < split_filesize * 1.1: + if file_size < split_filesize * 1.2: return if split_filesize and chunk_strategy == 'even': split_chunks = ffs.ceildiv(file_size, split_filesize) diff --git a/streamonitor/sites/chaturbate.py b/streamonitor/sites/chaturbate.py index 9891d5d..4768b9b 100644 --- a/streamonitor/sites/chaturbate.py +++ b/streamonitor/sites/chaturbate.py @@ -15,11 +15,16 @@ def getVideoUrl(self): return self.getWantedResolutionPlaylist(self.lastInfo['url']) def getStatus(self): - headers = {"X-Requested-With": "XMLHttpRequest"} + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/53' + '7.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36', + "X-Requested-With": "XMLHttpRequest" + } data = {"room_slug": self.username, "bandwidth": "high"} try: - r = requests.post("https://chaturbate.com/get_edge_hls_url_ajax/", headers=headers, data=data) + r = requests.post("https://chaturbate.com/get_edge_hls_url_ajax/", + headers=headers, data=data) self.lastInfo = r.json() if self.lastInfo["room_status"] == "public": diff --git a/streamonitor/sites/stripchat.py b/streamonitor/sites/stripchat.py index f0e3ce8..5285728 100644 --- a/streamonitor/sites/stripchat.py +++ b/streamonitor/sites/stripchat.py @@ -14,20 +14,23 @@ def getVideoUrl(self): )) def getStatus(self): - r = requests.get('https://stripchat.com/api/vr/v2/models/username/' + self.username, headers=self.headers) - if r.status_code != 200: + try: + r = requests.get('https://stripchat.com/api/vr/v2/models/username/' + self.username, headers=self.headers) + if r.status_code != 200: + return Bot.Status.UNKNOWN + + self.lastInfo = r.json() + + if self.lastInfo["model"]["status"] == "public" and self.lastInfo["isCamAvailable"] and self.lastInfo['cam']["isCamActive"]: + return Bot.Status.PUBLIC + if self.lastInfo["model"]["status"] in ["private", "groupShow", "p2p", "virtualPrivate", "p2pVoice"]: + return Bot.Status.PRIVATE + if self.lastInfo["model"]["status"] in ["off", "idle"]: + return Bot.Status.OFFLINE + self.logger.warn(f'Got unknown status: {self.lastInfo["model"]["status"]}') + return Bot.Status.UNKNOWN + except: return Bot.Status.UNKNOWN - - self.lastInfo = r.json() - - if self.lastInfo["model"]["status"] == "public" and self.lastInfo["isCamAvailable"] and self.lastInfo['cam']["isCamActive"]: - return Bot.Status.PUBLIC - if self.lastInfo["model"]["status"] in ["private", "groupShow", "p2p", "virtualPrivate", "p2pVoice"]: - return Bot.Status.PRIVATE - if self.lastInfo["model"]["status"] in ["off", "idle"]: - return Bot.Status.OFFLINE - self.logger.warn(f'Got unknown status: {self.lastInfo["model"]["status"]}') - return Bot.Status.UNKNOWN Bot.loaded_sites.add(StripChat) From 85787cbb08d815c119b6254543612a6065079bc8 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Fri, 2 Jun 2023 04:52:07 +0300 Subject: [PATCH 06/25] New WebGUI --- Downloader.py | 6 +- config.json | 92 +++++- requirements.txt | 3 +- streamonitor/managers/assets/app.css | 427 +++++++++++++++++++++++++++ streamonitor/managers/climanager.py | 4 +- streamonitor/managers/dashmanager.py | 172 +++++++++++ test.py | 4 + 7 files changed, 699 insertions(+), 9 deletions(-) create mode 100644 streamonitor/managers/assets/app.css create mode 100644 streamonitor/managers/dashmanager.py create mode 100644 test.py diff --git a/Downloader.py b/Downloader.py index fea95a4..51cfb52 100644 --- a/Downloader.py +++ b/Downloader.py @@ -4,11 +4,12 @@ from streamonitor.managers.httpmanager import HTTPManager from streamonitor.managers.climanager import CLIManager from streamonitor.managers.zmqmanager import ZMQManager +from streamonitor.managers.dashmanager import DashManager from streamonitor.managers.outofspace_detector import OOSDetector from streamonitor.clean_exit import CleanExit import streamonitor.sites # must have - + def is_docker(): path = '/proc/self/cgroup' return ( @@ -39,5 +40,8 @@ def main(): http_manager = HTTPManager(streamers) http_manager.start() + dash_manager = DashManager(streamers) + dash_manager.start() + main() diff --git a/config.json b/config.json index cf5e2b5..a53f032 100644 --- a/config.json +++ b/config.json @@ -294,11 +294,6 @@ "username": "aura_69", "running": true }, - { - "site": "Chaturbate", - "username": "marcussanchez", - "running": true - }, { "site": "BongaCams", "username": "Alisha008", @@ -531,7 +526,7 @@ }, { "site": "BongaCams", - "username": "candykushhh", + "username": "CandyKushhh", "running": true }, { @@ -553,5 +548,90 @@ "site": "BongaCams", "username": "HotDemonic", "running": true + }, + { + "site": "Chaturbate", + "username": "kyrrax23", + "running": true + }, + { + "site": "Chaturbate", + "username": "emmabraun", + "running": true + }, + { + "site": "Chaturbate", + "username": "_paprika", + "running": true + }, + { + "site": "Chaturbate", + "username": "_b_a_n_s_h_e_e_", + "running": true + }, + { + "site": "Chaturbate", + "username": "marcussanchez", + "running": true + }, + { + "site": "Chaturbate", + "username": "yourlovelyjul", + "running": true + }, + { + "site": "Chaturbate", + "username": "mollyflwers", + "running": true + }, + { + "site": "Chaturbate", + "username": "your_cute_marshmallow", + "running": true + }, + { + "site": "Chaturbate", + "username": "kotosqvad", + "running": true + }, + { + "site": "Chaturbate", + "username": "deeplily", + "running": true + }, + { + "site": "Chaturbate", + "username": "dose_of_sex", + "running": true + }, + { + "site": "Chaturbate", + "username": "angie_daniels", + "running": true + }, + { + "site": "Chaturbate", + "username": "frori2", + "running": true + }, + { + "site": "Chaturbate", + "username": "lolliruth", + "running": true + }, + { + "site": "Chaturbate", + "username": "projektmelody", + "running": true + }, + { + "site": "Chaturbate", + "username": "alexisward", + "running": true + }, + { + "site": "Chaturbate", + "username": "limy_fresh", + "running": true } ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d07a493..8ceb663 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ websocket-client ffmpy m3u8 colorama -tqdm \ No newline at end of file +tqdm +dash \ No newline at end of file diff --git a/streamonitor/managers/assets/app.css b/streamonitor/managers/assets/app.css new file mode 100644 index 0000000..32a5a05 --- /dev/null +++ b/streamonitor/managers/assets/app.css @@ -0,0 +1,427 @@ +/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */ + +/* Document + ========================================================================== */ + +/** + * 1. Correct the line height in all browsers. + * 2. Prevent adjustments of font size after orientation changes in iOS. + */ + +html { + line-height: 1.15; + /* 1 */ + -webkit-text-size-adjust: 100%; + /* 2 */ +} + +/* Sections + ========================================================================== */ + +/** + * Remove the margin in all browsers. + */ + +body { + margin: 0; +} + +/** + * Render the `main` element consistently in IE. + */ + +main { + display: block; +} + +/** + * Correct the font size and margin on `h1` elements within `section` and + * `article` contexts in Chrome, Firefox, and Safari. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +/* Grouping content + ========================================================================== */ + +/** + * 1. Add the correct box sizing in Firefox. + * 2. Show the overflow in Edge and IE. + */ + +hr { + box-sizing: content-box; + /* 1 */ + height: 0; + /* 1 */ + overflow: visible; + /* 2 */ +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +pre { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 1em; + /* 2 */ +} + +/* Text-level semantics + ========================================================================== */ + +/** + * Remove the gray background on active links in IE 10. + */ + +a { + background-color: transparent; +} + +/** + * 1. Remove the bottom border in Chrome 57- + * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. + */ + +abbr[title] { + border-bottom: none; + /* 1 */ + text-decoration: underline; + /* 2 */ + text-decoration: underline dotted; + /* 2 */ +} + +/** + * Add the correct font weight in Chrome, Edge, and Safari. + */ + +b, +strong { + font-weight: bolder; +} + +/** + * 1. Correct the inheritance and scaling of font size in all browsers. + * 2. Correct the odd `em` font sizing in all browsers. + */ + +code, +kbd, +samp { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 1em; + /* 2 */ +} + +/** + * Add the correct font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` elements from affecting the line height in + * all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sub { + bottom: -0.25em; +} + +sup { + top: -0.5em; +} + +/* Embedded content + ========================================================================== */ + +/** + * Remove the border on images inside links in IE 10. + */ + +img { + border-style: none; +} + +/* Forms + ========================================================================== */ + +/** + * 1. Change the font styles in all browsers. + * 2. Remove the margin in Firefox and Safari. + */ + +button, +input, +optgroup, +select, +textarea { + font-family: JetBrains Mono, monospace; + /* 1 */ + font-size: 100%; + /* 1 */ + line-height: 1.15; + /* 1 */ + margin: 0; + /* 2 */ +} + +/** + * Show the overflow in IE. + * 1. Show the overflow in Edge. + */ + +button, +input { + /* 1 */ + overflow: visible; +} + +/** + * Remove the inheritance of text transform in Edge, Firefox, and IE. + * 1. Remove the inheritance of text transform in Firefox. + */ + +button, +select { + /* 1 */ + text-transform: none; +} + +/** + * Correct the inability to style clickable types in iOS and Safari. + */ + +button, +[type="button"], +[type="reset"], +[type="submit"] { + -webkit-appearance: button; +} + +/** + * Remove the inner border and padding in Firefox. + */ + +button::-moz-focus-inner, +[type="button"]::-moz-focus-inner, +[type="reset"]::-moz-focus-inner, +[type="submit"]::-moz-focus-inner { + border-style: none; + padding: 0; +} + +/** + * Restore the focus styles unset by the previous rule. + */ + +button:-moz-focusring, +[type="button"]:-moz-focusring, +[type="reset"]:-moz-focusring, +[type="submit"]:-moz-focusring { + outline: 1px dotted ButtonText; +} + +/** + * Correct the padding in Firefox. + */ + +fieldset { + padding: 0.35em 0.75em 0.625em; +} + +/** + * 1. Correct the text wrapping in Edge and IE. + * 2. Correct the color inheritance from `fieldset` elements in IE. + * 3. Remove the padding so developers are not caught out when they zero out + * `fieldset` elements in all browsers. + */ + +legend { + box-sizing: border-box; + /* 1 */ + color: inherit; + /* 2 */ + display: table; + /* 1 */ + max-width: 100%; + /* 1 */ + padding: 0; + /* 3 */ + white-space: normal; + /* 1 */ +} + +/** + * Add the correct vertical alignment in Chrome, Firefox, and Opera. + */ + +progress { + vertical-align: baseline; +} + +/** + * Remove the default vertical scrollbar in IE 10+. + */ + +textarea { + overflow: auto; +} + +/** + * 1. Add the correct box sizing in IE 10. + * 2. Remove the padding in IE 10. + */ + +[type="checkbox"], +[type="radio"] { + box-sizing: border-box; + /* 1 */ + padding: 0; + /* 2 */ +} + +/** + * Correct the cursor style of increment and decrement buttons in Chrome. + */ + +[type="number"]::-webkit-inner-spin-button, +[type="number"]::-webkit-outer-spin-button { + height: auto; +} + +/** + * 1. Correct the odd appearance in Chrome and Safari. + * 2. Correct the outline style in Safari. + */ + +[type="search"] { + -webkit-appearance: textfield; + /* 1 */ + outline-offset: -2px; + /* 2 */ +} + +/** + * Remove the inner padding in Chrome and Safari on macOS. + */ + +[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * 1. Correct the inability to style clickable types in iOS and Safari. + * 2. Change font properties to `inherit` in Safari. + */ + +::-webkit-file-upload-button { + -webkit-appearance: button; + /* 1 */ + font: inherit; + /* 2 */ +} + +/* Interactive + ========================================================================== */ + +/* + * Add the correct display in Edge, IE 10+, and Firefox. + */ + +details { + display: block; +} + +/* + * Add the correct display in all browsers. + */ + +summary { + display: list-item; +} + +/* Misc + ========================================================================== */ + +/** + * Add the correct display in IE 10+. + */ + +template { + display: none; +} + +/** + * Add the correct display in IE 10. + */ + +[hidden] { + display: none; +} + +html { + box-sizing: border-box; + font-family: JetBrains Mono, monospace; + background-color: #191919; + color: #EEEEEE; + margin: 8px; +} + +h1 { + /* center text */ + text-align: center; +} + +/* div id=table */ +#table { + border-collapse: collapse; + border-spacing: 0; + /* center table */ + margin-left: auto; + margin-right: auto; + + /* table width */ + width: 70%; + max-width: 100%; + min-width: 10% +} + +#table { + background-color: #606060; + + /* shadow */ + box-shadow: 0 1px 1px rgba(255, 255, 255, 0.12), + 0 3px 3px rgba(255, 255, 255, 0.12), + 0 6px 6px rgba(255, 255, 255, 0.12), + 0 10px 10px rgba(255, 255, 255, 0.12), + 0 20px 20px rgba(255, 255, 255, 0.12); +} + +/* class column-header-name */ +.column-header-name { + /* center text */ + text-align: center; +} + +.dash-header { + background-color: #101010; +} \ No newline at end of file diff --git a/streamonitor/managers/climanager.py b/streamonitor/managers/climanager.py index f946276..c78ef7e 100644 --- a/streamonitor/managers/climanager.py +++ b/streamonitor/managers/climanager.py @@ -2,6 +2,7 @@ from streamonitor.manager import Manager from streamonitor.clean_exit import CleanExit import streamonitor.log as log +import colorama as color if sys.platform != "win32": import readline @@ -10,7 +11,8 @@ class CLIManager(Manager): def __init__(self, streamers): super().__init__(streamers) - self.logger = log.Logger("manager_cli") + self.logger = log.Logger(f"{color.Style.RESET_ALL}{color.Fore.CYAN}\ +manager_cli{color.Style.RESET_ALL}") def run(self): while True: diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py new file mode 100644 index 0000000..dd22971 --- /dev/null +++ b/streamonitor/managers/dashmanager.py @@ -0,0 +1,172 @@ +from flask import Flask, request +import os +from streamonitor.bot import Bot +import streamonitor.log as log +from streamonitor.manager import Manager +from dash import Dash +from dash import dcc +from dash import html +import plotly.express as px +import pandas as pd +from dash import dash_table +from dash import dependencies +from timeit import default_timer as timer +import time + + +class DashManager(Manager): + def __init__(self, streamers): + super().__init__(streamers) + self.logger = log.Logger("manager") + + def run(self): + app = Dash(__name__, title="CG-DL Status", update_title="CG-DL Status", + external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", + ]) + + # remove all css + app.css.config.serve_locally = False + + # set title + app.title = "CG-DL Status" + + def header(): + return "

CG-DL Status

" + + def scripts(): + pass + + def status(): + df = pd.DataFrame( + columns=["Site", "Username", "Started", "Status"]) + for streamer in self.streamers: + df = df._append({"Site": streamer.site, + "Username": streamer.username, + "Started": streamer.running, + "Status": streamer.status()}, ignore_index=True) + return df + + def recordings(): + streamer = self.getStreamer( + request.args.get("user"), request.args.get("site")) + try: + temp = [] + for elem in os.listdir("./downloads/{u} [{s}]".format(u=streamer.username, s=streamer.siteslug)): + temp.append(elem) + if temp == []: + return "No recordings" + else: + return temp + except: + return "No recordings" + + class colors: + green = '#3D9970' + yellow = '#FFDC00' + red = '#FF4136' + grey = '#DDDDDD' + white = '#EEEEEE' + black = '#191919' + + app.layout = html.Div( + [ + html.H1("CG-DL Status"), + dcc.Interval( + id='interval-component', + interval=20*1000, # in milliseconds + n_intervals=0 + ), + # sorting fucntionalities + dash_table.DataTable( + id='table', + columns=[{"name": i, "id": i} + for i in status().columns], + data=status().to_dict('records'), + style_cell={ + 'textAlign': 'left', + 'backgroundColor': '#303030', + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + style_data_conditional=[ + { + 'if': { + 'filter_query': '{Status} = "Channel online"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.green, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Private show"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Not running"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Unknown error"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.red, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream for a while"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.grey, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.grey, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + } + ], + style_header={ + 'backgroundColor': colors.black, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace', + 'fontWeight': 'bold' + + }, + sort_mode="multi", + sort_action="native", + filter_action="native") + ], + style={ + 'backgroundColor': colors.black, + 'color': colors.white, + # remove margin for better mobile experience + 'margin': '0px', + 'font-family': 'font-family: JetBrains Mono, monospace' + }) + + @app.callback( + dependencies.Output('table', 'data'), + dependencies.Input('interval-component', 'n_intervals')) + def update_data(timestamp): + return status().to_dict('records') + + app.run(host="127.0.0.2", port=5000) diff --git a/test.py b/test.py new file mode 100644 index 0000000..ef21334 --- /dev/null +++ b/test.py @@ -0,0 +1,4 @@ +from streamonitor.managers import dashmanager + +if __name__ == "__main__": + dashmanager.HTTPManager().run() From 75ea10bff81d284fa40c3e86606a5d07b43a4360 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Fri, 2 Jun 2023 05:34:28 +0300 Subject: [PATCH 07/25] Nice port for Dash --- streamonitor/managers/dashmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index dd22971..53b4d6f 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -169,4 +169,4 @@ class colors: def update_data(timestamp): return status().to_dict('records') - app.run(host="127.0.0.2", port=5000) + app.run(host="127.0.0.1", port=6969) From 8ae6bcef3c18a41caa82f45037cd696b6d6257b9 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Fri, 2 Jun 2023 10:36:42 +0300 Subject: [PATCH 08/25] Add disk space to WebGUI --- Downloader.py | 4 ++-- config.json | 10 +++++----- parameters.py | 2 +- streamonitor/managers/assets/app.css | 5 +++++ streamonitor/managers/dashmanager.py | 30 +++++++++++++++++++++++++++- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/Downloader.py b/Downloader.py index 51cfb52..f6be944 100644 --- a/Downloader.py +++ b/Downloader.py @@ -37,8 +37,8 @@ def main(): zmq_manager = ZMQManager(streamers) zmq_manager.start() - http_manager = HTTPManager(streamers) - http_manager.start() + # http_manager = HTTPManager(streamers) + # http_manager.start() dash_manager = DashManager(streamers) dash_manager.start() diff --git a/config.json b/config.json index a53f032..80bc0d8 100644 --- a/config.json +++ b/config.json @@ -99,11 +99,6 @@ "username": "FiOnee", "running": true }, - { - "site": "StripChat", - "username": "Kitty_Killer_", - "running": true - }, { "site": "StripChat", "username": "RoseMarceau_", @@ -633,5 +628,10 @@ "site": "Chaturbate", "username": "limy_fresh", "running": true + }, + { + "site": "StripChat", + "username": "Kitty_Love_cum", + "running": true } ] \ No newline at end of file diff --git a/parameters.py b/parameters.py index 843dbd9..114283f 100644 --- a/parameters.py +++ b/parameters.py @@ -1,5 +1,5 @@ DOWNLOADS_DIR = 'downloads' -MIN_FREE_DISK_PERCENT = 60.0 # in % +MIN_FREE_DISK_PERCENT = 40.0 # in % DEBUG = False # You can enter a number to select a specific height. diff --git a/streamonitor/managers/assets/app.css b/streamonitor/managers/assets/app.css index 32a5a05..13ed24a 100644 --- a/streamonitor/managers/assets/app.css +++ b/streamonitor/managers/assets/app.css @@ -424,4 +424,9 @@ h1 { .dash-header { background-color: #101010; +} + +h2 { + /* center text */ + text-align: center; } \ No newline at end of file diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index 53b4d6f..a93fbd7 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -12,6 +12,8 @@ from dash import dependencies from timeit import default_timer as timer import time +import shutil +import parameters class DashManager(Manager): @@ -68,9 +70,17 @@ class colors: white = '#EEEEEE' black = '#191919' + def get_disk_space(): + total, used, free = shutil.disk_usage("./") + return total, used, free + app.layout = html.Div( [ html.H1("CG-DL Status"), + html.H2( + f'Disk space: {round(get_disk_space()[1]/1024/1024/1024,2)}\ +/{round(get_disk_space()[0]/1024/1024/1024,2)} GB (limit=\ +{round(get_disk_space()[0]/1024*(100-parameters.MIN_FREE_DISK_PERCENT)/1024/1024/100,2)} GB)', id="disk_space"), dcc.Interval( id='interval-component', interval=20*1000, # in milliseconds @@ -116,6 +126,15 @@ class colors: 'color': colors.black, 'font-family': 'font-family: JetBrains Mono, monospace' }, + { + 'if': { + 'filter_query': '{Status} = "Rate limited"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, { 'if': { 'filter_query': '{Status} = "Unknown error"', @@ -169,4 +188,13 @@ class colors: def update_data(timestamp): return status().to_dict('records') - app.run(host="127.0.0.1", port=6969) + # update disk space + @app.callback( + dependencies.Output('disk_space', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_disk_space(timestamp): + return f'Disk space: {round(get_disk_space()[1]/1024/1024/1024,2)}\ +/{round(get_disk_space()[0]/1024/1024/1024,2)} GB (limit=\ +{round(get_disk_space()[0]/1024*(100-parameters.MIN_FREE_DISK_PERCENT)/1024/1024/100,2)} GB)' + + app.run(host="127.0.0.1", port=5001) From 60fc4adb2338fe382763a937e775b571fb309545 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Fri, 2 Jun 2023 14:53:11 +0300 Subject: [PATCH 09/25] Add Erroe on downloading to WebUI --- streamonitor/managers/dashmanager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index a93fbd7..2adb996 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -144,6 +144,15 @@ def get_disk_space(): 'color': colors.white, 'font-family': 'font-family: JetBrains Mono, monospace' }, + { + 'if': { + 'filter_query': '{Status} = "Error on downloading"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.red, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, { 'if': { 'filter_query': '{Status} = "No stream for a while"', From 02a4b9217ff4a4ec35718022222c84a91c5170ea Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:33:28 +0300 Subject: [PATCH 10/25] New thumbnail system --- config.json | 254 +++++++++++++++++++++++------------------------ mac.py | 29 ++++++ make_vcsis.py | 16 +++ requirements.txt | 27 ++--- 4 files changed, 187 insertions(+), 139 deletions(-) create mode 100644 make_vcsis.py diff --git a/config.json b/config.json index 80bc0d8..93b8fad 100644 --- a/config.json +++ b/config.json @@ -2,636 +2,636 @@ { "site": "StripChat", "username": "raingurl", - "running": true + "running": false }, { "site": "StripChat", "username": "Kralya_Smile", - "running": true + "running": false }, { "site": "StripChat", "username": "lika_lil", - "running": true + "running": false }, { "site": "StripChat", "username": "Misss_Vikk", - "running": true + "running": false }, { "site": "StripChat", "username": "Scops_owl", - "running": true + "running": false }, { "site": "StripChat", "username": "boobsybootie", - "running": true + "running": false }, { "site": "StripChat", "username": "kassablanca_", - "running": true + "running": false }, { "site": "StripChat", "username": "littlecarolina", - "running": true + "running": false }, { "site": "StripChat", "username": "moon_shy", - "running": true + "running": false }, { "site": "StripChat", "username": "Careful_i_bite", - "running": true + "running": false }, { "site": "StripChat", "username": "Ginger_Pie", - "running": true + "running": false }, { "site": "StripChat", "username": "BUNY_PRINCESS", - "running": true + "running": false }, { "site": "StripChat", "username": "kate_pierce", - "running": true + "running": false }, { "site": "StripChat", "username": "_Lory_Like_", - "running": true + "running": false }, { "site": "StripChat", "username": "Filla--", - "running": true + "running": false }, { "site": "StripChat", "username": "susu-dy", - "running": true + "running": false }, { "site": "StripChat", "username": "-meowpa-", - "running": true + "running": false }, { "site": "StripChat", "username": "MaryBrent", - "running": true + "running": false }, { "site": "StripChat", "username": "EroticNana", - "running": true + "running": false }, { "site": "StripChat", "username": "FiOnee", - "running": true + "running": false }, { "site": "StripChat", "username": "RoseMarceau_", - "running": true + "running": false }, { "site": "StripChat", "username": "Lana_Lee_", - "running": true + "running": false }, { "site": "StripChat", "username": "Rocks_Babies_", - "running": true + "running": false }, { "site": "StripChat", "username": "Kassy_Sun", - "running": true + "running": false }, { "site": "StripChat", "username": "Vilgelminaa", - "running": true + "running": false }, { "site": "StripChat", "username": "ChloeAttwood_", - "running": true + "running": false }, { "site": "StripChat", "username": "DaddysLittleSlutX", - "running": true + "running": false }, { "site": "StripChat", "username": "doll_joy", - "running": true + "running": false }, { "site": "StripChat", "username": "AkiShina", - "running": true + "running": false }, { "site": "StripChat", "username": "Tsumugi_M", - "running": true + "running": false }, { "site": "StripChat", "username": "JessAdams", - "running": true + "running": false }, { "site": "StripChat", "username": "YourMisano", - "running": true + "running": false }, { "site": "StripChat", "username": "TayaMeow", - "running": true + "running": false }, { "site": "StripChat", "username": "Eva_Bear", - "running": true + "running": false }, { "site": "StripChat", "username": "Badassloli_", - "running": true + "running": false }, { "site": "StripChat", "username": "cutiesuee", - "running": true + "running": false }, { "site": "StripChat", "username": "amaya_mori", - "running": true + "running": false }, { "site": "StripChat", "username": "aerri_lee", - "running": true + "running": false }, { "site": "StripChat", "username": "asuna_love", - "running": true + "running": false }, { "site": "StripChat", "username": "emma_moanss", - "running": true + "running": false }, { "site": "BongaCams", "username": "HollyTailor", - "running": true + "running": false }, { "site": "StripChat", "username": "Runa_star", - "running": true + "running": false }, { "site": "BongaCams", "username": "Pussy-", - "running": true + "running": false }, { "site": "BongaCams", "username": "MiaTonyy", - "running": true + "running": false }, { "site": "BongaCams", "username": "PlushCatt", - "running": true + "running": false }, { "site": "BongaCams", "username": "milky-cunt", - "running": true + "running": false }, { "site": "BongaCams", "username": "WhiteGirl-one", - "running": true + "running": false }, { "site": "BongaCams", "username": "Bestblondie", - "running": true + "running": false }, { "site": "BongaCams", "username": "-wellcum-", - "running": true + "running": false }, { "site": "BongaCams", "username": "MaxineDiaz", - "running": true + "running": false }, { "site": "BongaCams", "username": "Bestblondie", - "running": true + "running": false }, { "site": "Chaturbate", "username": "donnaayanami", - "running": true + "running": false }, { "site": "Chaturbate", "username": "vixenp", - "running": true + "running": false }, { "site": "Chaturbate", "username": "alice_kosmos", - "running": true + "running": false }, { "site": "Chaturbate", "username": "beka_hernandez", - "running": true + "running": false }, { "site": "Chaturbate", "username": "eva_elfyy", - "running": true + "running": false }, { "site": "Chaturbate", "username": "francesdonna", - "running": true + "running": false }, { "site": "Chaturbate", "username": "aura_69", - "running": true + "running": false }, { "site": "BongaCams", "username": "Alisha008", - "running": true + "running": false }, { "site": "BongaCams", "username": "Kittenmeew", - "running": true + "running": false }, { "site": "BongaCams", "username": "AbsintheGirl", - "running": true + "running": false }, { "site": "BongaCams", "username": "MeyShan", - "running": true + "running": false }, { "site": "BongaCams", "username": "NeNaSuTnA", - "running": true + "running": false }, { "site": "BongaCams", "username": "MarinnaCat", - "running": true + "running": false }, { "site": "BongaCams", "username": "KiraLiLime", - "running": true + "running": false }, { "site": "StripChat", "username": "SamanthaMorgann", - "running": true + "running": false }, { "site": "StripChat", "username": "MadissonSoft", - "running": true + "running": false }, { "site": "StripChat", "username": "ummpaLummpa", - "running": true + "running": false }, { "site": "StripChat", "username": "wendy_taylor_", - "running": true + "running": false }, { "site": "BongaCams", "username": "barsaa7", - "running": true + "running": false }, { "site": "BongaCams", "username": "Arisha-w", - "running": true + "running": false }, { "site": "BongaCams", "username": "Effy_S", - "running": true + "running": false }, { "site": "BongaCams", "username": "SoiaPenka", - "running": true + "running": false }, { "site": "BongaCams", "username": "ElizabethGlor", - "running": true + "running": false }, { "site": "BongaCams", "username": "Kralya-Smile", - "running": true + "running": false }, { "site": "BongaCams", "username": "LilyaLilac", - "running": true + "running": false }, { "site": "BongaCams", "username": "cyberannita", - "running": true + "running": false }, { "site": "BongaCams", "username": "Baby-Tisha", - "running": true + "running": false }, { "site": "BongaCams", "username": "Annelitt", - "running": true + "running": false }, { "site": "BongaCams", "username": "AmeliaLean", - "running": true + "running": false }, { "site": "BongaCams", "username": "syndicete", - "running": true + "running": false }, { "site": "BongaCams", "username": "MiaandMelissa", - "running": true + "running": false }, { "site": "BongaCams", "username": "EvaSSly", - "running": true + "running": false }, { "site": "BongaCams", "username": "Aru-Koto", - "running": true + "running": false }, { "site": "BongaCams", "username": "MariawwEmma", - "running": true + "running": false }, { "site": "BongaCams", "username": "LISA-ALYSSA", - "running": true + "running": false }, { "site": "BongaCams", "username": "JennaStroker", - "running": true + "running": false }, { "site": "BongaCams", "username": "FreyaAdams", - "running": true + "running": false }, { "site": "BongaCams", "username": "Lilit1201", - "running": true + "running": false }, { "site": "Chaturbate", "username": "fiassquad", - "running": true + "running": false }, { "site": "Chaturbate", "username": "ann_lover016", - "running": true + "running": false }, { "site": "Chaturbate", "username": "missnicolette1", - "running": true + "running": false }, { "site": "Chaturbate", "username": "hornybunnys", - "running": true + "running": false }, { "site": "Chaturbate", "username": "lollypetitte_", - "running": true + "running": false }, { "site": "Chaturbate", "username": "fairyinthewild", - "running": true + "running": false }, { "site": "Chaturbate", "username": "pornxxxcouple", - "running": true + "running": false }, { "site": "Chaturbate", "username": "sweetpornface", - "running": true + "running": false }, { "site": "Chaturbate", "username": "santiago_and_lana", - "running": true + "running": false }, { "site": "Chaturbate", "username": "angelandfairieskink", - "running": true + "running": false }, { "site": "BongaCams", "username": "Alastaira", - "running": true + "running": false }, { "site": "BongaCams", "username": "Vasilisa-V", - "running": true + "running": false }, { "site": "BongaCams", "username": "ElisBraun", - "running": true + "running": false }, { "site": "BongaCams", "username": "Diafoxy", - "running": true + "running": false }, { "site": "BongaCams", "username": "Lola3000", - "running": true + "running": false }, { "site": "BongaCams", "username": "CandyKushhh", - "running": true + "running": false }, { "site": "BongaCams", "username": "KateDanverr", - "running": true + "running": false }, { "site": "BongaCams", "username": "crispy-tati-", - "running": true + "running": false }, { "site": "BongaCams", "username": "MissCatty", - "running": true + "running": false }, { "site": "BongaCams", "username": "HotDemonic", - "running": true + "running": false }, { "site": "Chaturbate", "username": "kyrrax23", - "running": true + "running": false }, { "site": "Chaturbate", "username": "emmabraun", - "running": true + "running": false }, { "site": "Chaturbate", "username": "_paprika", - "running": true + "running": false }, { "site": "Chaturbate", "username": "_b_a_n_s_h_e_e_", - "running": true + "running": false }, { "site": "Chaturbate", "username": "marcussanchez", - "running": true + "running": false }, { "site": "Chaturbate", "username": "yourlovelyjul", - "running": true + "running": false }, { "site": "Chaturbate", "username": "mollyflwers", - "running": true + "running": false }, { "site": "Chaturbate", "username": "your_cute_marshmallow", - "running": true + "running": false }, { "site": "Chaturbate", "username": "kotosqvad", - "running": true + "running": false }, { "site": "Chaturbate", "username": "deeplily", - "running": true + "running": false }, { "site": "Chaturbate", "username": "dose_of_sex", - "running": true + "running": false }, { "site": "Chaturbate", "username": "angie_daniels", - "running": true + "running": false }, { "site": "Chaturbate", "username": "frori2", - "running": true + "running": false }, { "site": "Chaturbate", "username": "lolliruth", - "running": true + "running": false }, { "site": "Chaturbate", "username": "projektmelody", - "running": true + "running": false }, { "site": "Chaturbate", "username": "alexisward", - "running": true + "running": false }, { "site": "Chaturbate", "username": "limy_fresh", - "running": true + "running": false }, { "site": "StripChat", "username": "Kitty_Love_cum", - "running": true + "running": false } ] \ No newline at end of file diff --git a/mac.py b/mac.py index 77fd10a..fda8d0c 100644 --- a/mac.py +++ b/mac.py @@ -3,6 +3,7 @@ import colorama as col import threading as th from tqdm import tqdm +import make_vcsis from src import move_all as ma from src import splitter as sp @@ -60,7 +61,35 @@ def call_function(filename): DONE!{col.Style.RESET_ALL}') +def vcsis(foldername: str = 'videos', overwrite: bool = False): + """ + Create a video contact sheet image for each video file in foldername. + + Parameters + ---------- + foldername (str): name of folder containing video files + overwrite (bool): overwrite existing vcsi files, default is False + """ + for root, dirs, filenames in os.walk(foldername): + for filename in tqdm(filenames, desc=f'{col.Style.RESET_ALL}\ +[VCSI]: Making thumbnails'): + if filename.endswith(('.mp4', '.mkv', '.avi', '.mov')): + try: + make_vcsis.main(foldername + '/' + + filename, overwrite=overwrite) + except Exception as e: + print( + f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} {e}') + except KeyboardInterrupt: + print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.RED}\ +ERROR:{col.Style.RESET_ALL} Keyboard interrupt') + print(f'{col.Style.RESET_ALL}[VCSI]: {col.Fore.GREEN}\ +DONE!{col.Style.RESET_ALL}') + + if __name__ == '__main__': move_and_split('downloads', 'videos', file_size=2000000000) dl.delete_less('videos', 200000000, log=True) + vcsis('videos', overwrite=False) input() diff --git a/make_vcsis.py b/make_vcsis.py new file mode 100644 index 0000000..0c151a6 --- /dev/null +++ b/make_vcsis.py @@ -0,0 +1,16 @@ +import os +import sys +import subprocess as sub +import vcsi + + +def main(filename: str = 'videos', overwrite: bool = False): + args = ["vcsi", "-t", "-w 7680", "-g 8x8"] + if not overwrite: + args.append("--no-overwrite") + args.append(filename) + cp = sub.run(args) + + +if __name__ == '__main__': + main('videos/AbsintheGirl-20230609-121937.mp4') diff --git a/requirements.txt b/requirements.txt index 8ceb663..c3fe62a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,15 @@ -requests -terminaltables -pyzmq -flask -termcolor -beautifulsoup4 -websocket-client -ffmpy -m3u8 -colorama -tqdm -dash \ No newline at end of file +requests>=2.31.0 +terminaltables>=3.1.10 +pyzmq>=25.1.0 +flask>=2.3.2 +termcolor>=2.3.0 +beautifulsoup4>=4.12.2 +websocket-client>=1.5.2 +ffmpy>=0.3.0 +m3u8>=3.5.0 +colorama>=0.4.6 +tqdm>=4.65.0 +dash>=2.0.0 +numpy>=1.24.3 +pandas>=2.0.2 +vcsi>=7.0.15 \ No newline at end of file From c7b6a483fab4f3358e3992882ecfe2fb1233f48a Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:39:13 +0300 Subject: [PATCH 11/25] README update --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 88efbd3..cbb4460 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # StreaMonitor -A Python3 application for monitoring and saving (mostly adult) live streams from various websites. +A Python application for monitoring and saving (mostly adult) live streams from various websites. -Inspired by [Recordurbate](https://github.com/oliverjrose99/Recordurbate) +Inspired by: [Recordurbate](https://github.com/oliverjrose99/Recordurbate) +Thumbnail system code by: [VCSI](https://github.com/amietn/vcsi) ## Supported sites | Site name | Abbreviation | Aliases | Quirks | Selectable resolution | @@ -40,11 +41,12 @@ The application has the following interfaces: * Web interface (only status) #### Starting and console -Start the downloader (it does not fork yet)\ +Start the downloader with the following command. Automatically imports all streamers from the config file. ``` -python3 Downloader.py +python Downloader.py ``` +(On Windows you can use the `run.bat` file) On the console you can use the following commands: ``` From e5fab4bc5dabe7102c02d3d5e8a5f8f9f1adae20 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:40:38 +0300 Subject: [PATCH 12/25] README update --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cbb4460..66c158a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # StreaMonitor A Python application for monitoring and saving (mostly adult) live streams from various websites. -Inspired by: [Recordurbate](https://github.com/oliverjrose99/Recordurbate) -Thumbnail system code by: [VCSI](https://github.com/amietn/vcsi) +# Credits +- Inspired by: [Recordurbate](https://github.com/oliverjrose99/Recordurbate) +- Thumbnail system code by: [VCSI](https://github.com/amietn/vcsi) ## Supported sites | Site name | Abbreviation | Aliases | Quirks | Selectable resolution | From 62817fd663f6cdd06278ab42ed076a4d3f395c53 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:43:23 +0300 Subject: [PATCH 13/25] README update --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 66c158a..30f0571 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,10 @@ There are hundreds of clones of the sites above, you can read about them on [thi ## Requirements * Python 3 - * Install packages listed in requirements.txt with pip. + * Install packages listed in requirements.txt: + ``` + pip install -r requirements.txt + ``` * FFmpeg ## Usage @@ -41,7 +44,7 @@ The application has the following interfaces: * External console via ZeroMQ (sort of working) * Web interface (only status) -#### Starting and console +#### Starting Start the downloader with the following command. Automatically imports all streamers from the config file. ``` @@ -49,6 +52,7 @@ python Downloader.py ``` (On Windows you can use the `run.bat` file) +#### Console On the console you can use the following commands: ``` add - Add streamer to the list (also starts monitoring) From bc292111b5d2c72f978e8721837e10e565ba2719 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:44:42 +0300 Subject: [PATCH 14/25] README update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30f0571..094c22e 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ There are hundreds of clones of the sites above, you can read about them on [thi ``` pip install -r requirements.txt ``` -* FFmpeg +* [FFmpeg](https://ffmpeg.org/download.html) ## Usage From 5525d994573c809a2b5a9d0218bfdb7ffeb8b8b3 Mon Sep 17 00:00:00 2001 From: Kseen715 Date: Tue, 27 Jun 2023 10:50:55 +0300 Subject: [PATCH 15/25] README update --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 094c22e..74c7bfb 100644 --- a/README.md +++ b/README.md @@ -73,32 +73,32 @@ For the `site` input, you can use either the full or the short format of the sit #### "Remote" controller Add or remove a streamer to record (Also saves config file) ``` -python3 Controller.py add -python3 Controller.py remove +python Controller.py add +python Controller.py remove ``` Start/stop recording streamers ``` -python3 Controller.py +python Controller.py ``` List the streamers in the config ``` -python3 Controller.py status +python Controller.py status ``` #### Web interface You can access the web interface on port 5000. It just prints the same information as the status command. -You can also get a list of the recorded streams. +~~You can also get a list of the recorded streams.~~ Further improvements can be expected. -## Docker support +## ~~Docker support~~ -You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use. -Simply start it in the folder with `docker-compose up`. +~~You can run this application in docker. I prefer docker-compose so I included an example docker-compose.yml file that you can use.~~ +~~Simply start it in the folder with `docker-compose up`.~~ ## Configuration From 3a4654031ed9cb4f9c7022fcdfc8b8d95c4c1a39 Mon Sep 17 00:00:00 2001 From: kseen Date: Tue, 29 Aug 2023 14:04:27 +0300 Subject: [PATCH 16/25] Old changes --- parameters.py | 2 +- streamonitor/managers/assets/app.css | 291 +++++++++++++++++-- streamonitor/managers/dashmanager.py | 416 ++++++++++++++++++--------- web-lexicon.json | 21 ++ web-settings.json | 3 + 5 files changed, 574 insertions(+), 159 deletions(-) create mode 100644 web-lexicon.json create mode 100644 web-settings.json diff --git a/parameters.py b/parameters.py index 114283f..971a270 100644 --- a/parameters.py +++ b/parameters.py @@ -1,5 +1,5 @@ DOWNLOADS_DIR = 'downloads' -MIN_FREE_DISK_PERCENT = 40.0 # in % +MIN_FREE_DISK_PERCENT = 10.0 # in % DEBUG = False # You can enter a number to select a specific height. diff --git a/streamonitor/managers/assets/app.css b/streamonitor/managers/assets/app.css index 13ed24a..18897cc 100644 --- a/streamonitor/managers/assets/app.css +++ b/streamonitor/managers/assets/app.css @@ -378,42 +378,156 @@ template { display: none; } +/* END OF RESET.CSS */ + +/* START OF STYLE.CSS */ + +:root { + --color-green-light: #50c590; + --color-green: #3D9970; + --color-green-dark: #2b6a4d; + --color-yellow-light: #ffe53a; + --color-yellow: #FFDC00; + --color-yellow-dark: #bba200; + --color-red-light: #e46c65; + --color-red: #FF4136; + --color-red-dark: #c0352d; + + --color-white-light: #FFFFFF; + --color-white: #EEEEEE; + --color-grey-light: #a1a1a1; + --color-grey: #606060; + --color-grey-dark: #2f2f2f; + --color-black: #191919; + --color-black-dark: #101010; + + --color-background-light: var(--color-grey); + --color-background: var(--color-black); + --color-background-dark: var(--color-black-dark); + --color-text: var(--color-white); + --color-border: var(--color-grey-light); + + --font-jet-brains: JetBrains Mono, monospace; + + --color-shadow: rgba(108, 108, 108, 0.04); + --shadow: + 2px 1px 1px var(--color-shadow), + 2px 2px 2px var(--color-shadow), + 2px 3px 3px var(--color-shadow), + 2px 4px 4px var(--color-shadow), + 2px 5px 5px var(--color-shadow), + 2px 6px 6px var(--color-shadow), + 2px 7px 7px var(--color-shadow); +} + +div#main_div { + margin-left: 15%; + margin-right: 15%; + margin-top: 0; + margin-bottom: 0; + + font-family: var(--font-jet-brains); +} + html { box-sizing: border-box; - font-family: JetBrains Mono, monospace; - background-color: #191919; - color: #EEEEEE; - margin: 8px; + font-family: var(--jet-brains-font); + background-color: var(--color-background); + color: var(--color-text); } h1 { - /* center text */ text-align: center; } -/* div id=table */ -#table { - border-collapse: collapse; - border-spacing: 0; - /* center table */ - margin-left: auto; - margin-right: auto; +h2 { + text-align: center; +} - /* table width */ - width: 70%; - max-width: 100%; - min-width: 10% +h3 { + margin-top: 0; + margin-bottom: 5px; + text-align: left; +} + +h4 { + margin-top: 0; + text-align: left; +} + +button { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; + + text-align: right; + + padding-top: 7px; + padding-bottom: 7px; + padding-left: 20px; + padding-right: 20px; + + min-width: 150px; + min-height: 40px; + + text-align: center; + + background-color: var(--color-background); + color: var(--color-text); + + box-shadow: var(--shadow); +} + +button:hover { + background: var(--color-background-dark); + color: var(--color-text); } -#table { - background-color: #606060; +button#start_all, +button#add_streamer { + background-color: var(--color-green); + margin-left: 10px; +} + +button#start_all:hover, +button#add_streamer:hover { + background: var(--color-green-dark); +} + +button#stop_all, +button#remove_streamer { + background-color: var(--color-red); +} - /* shadow */ - box-shadow: 0 1px 1px rgba(255, 255, 255, 0.12), - 0 3px 3px rgba(255, 255, 255, 0.12), - 0 6px 6px rgba(255, 255, 255, 0.12), - 0 10px 10px rgba(255, 255, 255, 0.12), - 0 20px 20px rgba(255, 255, 255, 0.12); +button#stop_all:hover, +button#remove_streamer:hover { + background: var(--color-red-dark); +} + +div#buttons-all-streamers, +div#buttons-single-streamer { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; +} + +div#table { + border-collapse: collapse; + border-spacing: 0; + margin-top: 30px; + margin-bottom: 300px; + + background-color: var(--color-background-light); + box-shadow: var(--shadow); + color: var(--color-text); + font-family: var(--font-jet-brains); } /* class column-header-name */ @@ -423,10 +537,133 @@ h1 { } .dash-header { - background-color: #101010; + background-color: var(--color-background-dark); } -h2 { - /* center text */ + +input { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; + + background-color: var(--color-background-light); + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +input#username { + width: 300px; + height: 36px; + max-height: 36px; + min-height: 36px; + text-align: left; + + margin-right: 10px; + + color: var(--color-white); + font-family: var(--font-jet-brains); + + padding-left: 10px; + padding-right: 10px; + + box-shadow: var(--shadow); +} + +::placeholder { + color: var(--color-grey-light); +} + +div#user-manage { + display: grid; + /* grid to the right */ + grid-template-columns: 1fr 1fr; + grid-template-rows: 1fr 1fr; + + margin: 0; + padding: 0; + margin-top: 40px; + margin-bottom: 10px; + height: 40px; + text-align: left; + font-family: var(--font-jet-brains); +} + +div#console_div { + margin-top: 20px; + margin-bottom: 30px; + max-height: 200px; + min-height: 200px; + + padding: 10px; + + background-color: var(--color-background-dark); + color: var(--color-text); + + font-family: var(--font-jet-brains); + box-shadow: var(--shadow); + + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +div#credits, +div#versions { + margin-top: 20px; + margin-bottom: 20px; text-align: center; + font-family: var(--font-jet-brains); +} + +div#credit-version-box { + background-color: var(--color-background-dark); + + padding-top: 20px; + padding-left: 20px; + padding-right: 20px; + padding-bottom: 30px; + + border-color: var(--color-border); + border-width: 2px; + border-left: 0; + border-right: 0; + border-bottom: 0; + border-style: inset; + + box-shadow: var(--shadow); +} + +div#site.dash-dropdown { + margin-top: 0; + margin-bottom: 0; + margin-left: 20px; + margin-right: 0; + + background-color: var(--color-background-light); + min-height: 40px; + max-height: 40px; + height: 40px; + border-width: 2px; + border-radius: 10px; + border-color: var(--color-border); + border-style: inset; +} + +div.Select-control { + margin-left: 0; + margin-right: 0; + margin-top: 0; + margin-bottom: 0; +} + +div.Select { + margin-left: 0; + margin-right: 0; + margin-top: 0; + margin-bottom: 0; } \ No newline at end of file diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index 2adb996..ad2c9f4 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -14,6 +14,30 @@ import time import shutil import parameters +import psutil +import json + + +def load_json(filename: str): + with open(filename, "r") as f: + return json.load(f) + + +def load_settings(): + settings = load_json("web-settings.json") + return settings + + +def load_lexicon(): + lex = load_json("web-lexicon.json") + return lex + + +UPDATE_INTERVAL = 1 # seconds + +# Lexicon for the web interface +lex = load_lexicon() +print(lex) class DashManager(Manager): @@ -22,30 +46,27 @@ def __init__(self, streamers): self.logger = log.Logger("manager") def run(self): - app = Dash(__name__, title="CG-DL Status", update_title="CG-DL Status", + app = Dash(__name__, title=lex['title'], update_title=None, external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", ]) - # remove all css app.css.config.serve_locally = False - # set title - app.title = "CG-DL Status" - - def header(): - return "

CG-DL Status

" + app.title = lex['title'] def scripts(): pass def status(): df = pd.DataFrame( - columns=["Site", "Username", "Started", "Status"]) + columns=[lex['site'], lex['username'], + lex['started'], lex['status']]) for streamer in self.streamers: - df = df._append({"Site": streamer.site, - "Username": streamer.username, - "Started": streamer.running, - "Status": streamer.status()}, ignore_index=True) + df = df._append({lex['site']: streamer.site, + lex['username']: streamer.username, + lex['started']: streamer.running, + lex['status']: streamer.status()}, + ignore_index=True) return df def recordings(): @@ -53,7 +74,10 @@ def recordings(): request.args.get("user"), request.args.get("site")) try: temp = [] - for elem in os.listdir("./downloads/{u} [{s}]".format(u=streamer.username, s=streamer.siteslug)): + for elem \ + in os.listdir("./downloads/{u} [{s}]". + format(u=streamer.username, + s=streamer.siteslug)): temp.append(elem) if temp == []: return "No recordings" @@ -70,126 +94,222 @@ class colors: white = '#EEEEEE' black = '#191919' - def get_disk_space(): - total, used, free = shutil.disk_usage("./") - return total, used, free - app.layout = html.Div( [ - html.H1("CG-DL Status"), - html.H2( - f'Disk space: {round(get_disk_space()[1]/1024/1024/1024,2)}\ -/{round(get_disk_space()[0]/1024/1024/1024,2)} GB (limit=\ -{round(get_disk_space()[0]/1024*(100-parameters.MIN_FREE_DISK_PERCENT)/1024/1024/100,2)} GB)', id="disk_space"), - dcc.Interval( - id='interval-component', - interval=20*1000, # in milliseconds - n_intervals=0 - ), - # sorting fucntionalities - dash_table.DataTable( - id='table', - columns=[{"name": i, "id": i} - for i in status().columns], - data=status().to_dict('records'), - style_cell={ - 'textAlign': 'left', - 'backgroundColor': '#303030', - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - style_data_conditional=[ - { - 'if': { - 'filter_query': '{Status} = "Channel online"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.green, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Private show"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Not running"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Rate limited"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Unknown error"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.red, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Error on downloading"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.red, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "No stream for a while"', - 'column_id': 'Status' + html.Div( + [ + html.H1(lex['title']), + + html.Div( + [ + html.Button( + lex['stop_all'], + id='stop_all', + n_clicks=0 + ), + html.Button( + lex['start_all'], + id='start_all', + n_clicks=0 + ), + ], + id="buttons-all-streamers" + ), + # field to enter username + html.Div(children=[ + dcc.Input( + id="username", + type="text", + placeholder=lex['username'], + value="", + style={ + 'max-height': '36px', + 'height': '36px', + }, + ), + dcc.Dropdown( + id="site", + options=[ + {'label': html.Span(['Chaturbate'], style={ + 'background-color': colors.green}), + 'value': 'chaturbate'}, + {'label': 'OnlyFans', 'value': 'onlyfans'}], + className="site-dropdown", + placeholder=lex['site'], + style={ + 'width': '100px', + 'min-height': '30px', + 'height': '40px', + 'margin': 0, + 'padding': 0, + # 'margin-left': '20px', + 'width': '200px', + 'border-radius': '10px', + 'text-align': 'left', + 'font-family': 'font-family: JetBrains Mono, monospace', + 'background-color': '#606060', + }, + ) + ], + id="user-manage" + ), + + html.Div( + [ + html.Button( + lex['remove'], + id='remove_streamer', + n_clicks=0 + ), + html.Button( + lex['add'], + id='add_streamer', + n_clicks=0 + ), + ], + id="buttons-single-streamer" + ), + + # console output text box + html.Div( + [ + html.Pre( + id='console', + children=lex['console_placeholder'], + ) + ], + id="console_div" + ), + + html.H3('{} . . .'.format( + lex['loading']), id="running"), + html.H3('{} . . .'.format( + lex['loading']), id="cpu_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="ram_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="disk_space"), + + dcc.Interval( + id='interval-component', + interval=UPDATE_INTERVAL*1000, # in milliseconds + n_intervals=0 + ), + # sorting fucntionalities + dash_table.DataTable( + id='table', + columns=[{"name": i, "id": i} + for i in status().columns], + data=status().to_dict('records'), + style_cell={ + 'textAlign': 'left', + 'backgroundColor': '#2f2f2f', + 'color': '#EEEEEE', + 'font-family': 'font-family: JetBrains Mono, monospace' }, - 'backgroundColor': colors.grey, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "No stream"', - 'column_id': 'Status' + style_data_conditional=[ + { + 'if': { + 'filter_query': '{Status} = "Channel online"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.green, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Private show"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Not running"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Rate limited"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.yellow, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Unknown error"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.red, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Error on downloading"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.red, + 'color': colors.white, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream for a while"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.grey, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream"', + 'column_id': 'Status' + }, + 'backgroundColor': colors.grey, + 'color': colors.black, + 'font-family': 'font-family: JetBrains Mono, monospace' + } + ], + style_header={ + 'backgroundColor': '#101010', + 'color': '#EEEEEE', + 'font-family': 'font-family: JetBrains Mono, monospace', + 'fontWeight': 'bold' }, - 'backgroundColor': colors.grey, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - } + sort_mode="multi", + sort_action="native", + filter_action="native"), ], - style_header={ - 'backgroundColor': colors.black, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace', - 'fontWeight': 'bold' - - }, - sort_mode="multi", - sort_action="native", - filter_action="native") - ], - style={ - 'backgroundColor': colors.black, - 'color': colors.white, - # remove margin for better mobile experience - 'margin': '0px', - 'font-family': 'font-family: JetBrains Mono, monospace' - }) + id="main_div"), + + html.Div( + [ + html.Div( + html.A(lex['credits']), + id="credits", + ), + html.Div( + html.A(lex['versions']), + id="versions", + ), + ], + id="credit-version-box", + ), + ] + ) + + # update table @app.callback( dependencies.Output('table', 'data'), @@ -197,13 +317,47 @@ def get_disk_space(): def update_data(timestamp): return status().to_dict('records') + # update running + @app.callback( + dependencies.Output('running', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_running(timestamp): + return '=> {}: {}/{}'.\ + format(lex['running'], + len(status()[status()["Status"] == + "Channel online"]), len(status())) + + # update cpu usage + @app.callback( + dependencies.Output('cpu_usage', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_cpu_usage(timestamp): + return '=> {}: {}%'.format(lex['cpu'], psutil.cpu_percent()) + + # update ram usage + @app.callback( + dependencies.Output('ram_usage', 'children'), + dependencies.Input('interval-component', 'n_intervals')) + def update_ram_usage(timestamp): + return '=> {}: {}/{} GB'.\ + format(lex['ram'], + round(psutil.virtual_memory()[3]/1024/1024/1024, 2), + round(psutil.virtual_memory()[0]/1024/1024/1024, 2)) + # update disk space @app.callback( dependencies.Output('disk_space', 'children'), dependencies.Input('interval-component', 'n_intervals')) def update_disk_space(timestamp): - return f'Disk space: {round(get_disk_space()[1]/1024/1024/1024,2)}\ -/{round(get_disk_space()[0]/1024/1024/1024,2)} GB (limit=\ -{round(get_disk_space()[0]/1024*(100-parameters.MIN_FREE_DISK_PERCENT)/1024/1024/100,2)} GB)' + return '=> {}: {}/{} {} ({} = {} {})'.\ + format(lex['harddrive'], + round(shutil.disk_usage("./")[1]/1024/1024/1024, 2), + round(shutil.disk_usage("./")[0]/1024/1024/1024, 2), + lex['gigabyte'], + lex['limit'], + round(shutil.disk_usage("./")[0]/1024 * + (100-parameters.MIN_FREE_DISK_PERCENT) / + 1024/1024/100, 2), + lex['gigabyte']) app.run(host="127.0.0.1", port=5001) diff --git a/web-lexicon.json b/web-lexicon.json new file mode 100644 index 0000000..df1d2dd --- /dev/null +++ b/web-lexicon.json @@ -0,0 +1,21 @@ +{ + "title": "StreaMonitor", + "remove": "Remove", + "add": "Add", + "stop_all": "Stop all", + "start_all": "Start all", + "username": "Username", + "site": "Site", + "status": "Status", + "started": "Started", + "console_placeholder": "Console output will appear here", + "loading": "Loading", + "running": "Running", + "cpu": "CPU", + "ram": "RAM", + "harddrive": "Disk", + "gigabyte": "GB", + "limit": "Limit", + "credits": "Kseen715@git", + "versions": "Plotly-5.14.1 Dash-2.9.3" +} \ No newline at end of file diff --git a/web-settings.json b/web-settings.json new file mode 100644 index 0000000..544b7b4 --- /dev/null +++ b/web-settings.json @@ -0,0 +1,3 @@ +{ + +} \ No newline at end of file From 40ae36f4c29701880c2eccf0cba8b125ef517477 Mon Sep 17 00:00:00 2001 From: kseen Date: Tue, 29 Aug 2023 21:58:38 +0300 Subject: [PATCH 17/25] Multi-page system --- config/web-config.json | 12 + web-lexicon.json => config/web-lexicon.json | 3 +- requirements.txt | 3 +- streamonitor/managers/assets/app.css | 38 ++- streamonitor/managers/dash_pages/home.py | 224 ++++++++++++++ streamonitor/managers/dash_pages/v_manager.py | 17 ++ streamonitor/managers/dashmanager.py | 278 ++---------------- streamonitor/managers/jsonmanager.py | 17 ++ web-settings.json | 3 - 9 files changed, 332 insertions(+), 263 deletions(-) create mode 100644 config/web-config.json rename web-lexicon.json => config/web-lexicon.json (86%) create mode 100644 streamonitor/managers/dash_pages/home.py create mode 100644 streamonitor/managers/dash_pages/v_manager.py create mode 100644 streamonitor/managers/jsonmanager.py delete mode 100644 web-settings.json diff --git a/config/web-config.json b/config/web-config.json new file mode 100644 index 0000000..6eb2cdd --- /dev/null +++ b/config/web-config.json @@ -0,0 +1,12 @@ +{ + "update_interval_s": 5, + "color_positive": "#3D9970", + "color_warning": "#FFDC00", + "color_negative": "#FF4136", + "color_neutral": "#DDDDDD", + "color_text": "#EEEEEE", + "color_text_inv": "#191919", + "color_dropdown_background": "#606060", + "color_cell_background": "#2f2f2f", + "color_table_header_background": "#101010" +} \ No newline at end of file diff --git a/web-lexicon.json b/config/web-lexicon.json similarity index 86% rename from web-lexicon.json rename to config/web-lexicon.json index df1d2dd..b487bf2 100644 --- a/web-lexicon.json +++ b/config/web-lexicon.json @@ -17,5 +17,6 @@ "gigabyte": "GB", "limit": "Limit", "credits": "Kseen715@git", - "versions": "Plotly-5.14.1 Dash-2.9.3" + "versions": "Plotly-5.14.1 Dash-2.9.3", + "v_manager": "V-Manager" } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c3fe62a..d0901f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ tqdm>=4.65.0 dash>=2.0.0 numpy>=1.24.3 pandas>=2.0.2 -vcsi>=7.0.15 \ No newline at end of file +vcsi>=7.0.15 +dash_bootstrap_components \ No newline at end of file diff --git a/streamonitor/managers/assets/app.css b/streamonitor/managers/assets/app.css index 18897cc..9b705a4 100644 --- a/streamonitor/managers/assets/app.css +++ b/streamonitor/managers/assets/app.css @@ -455,11 +455,12 @@ h4 { text-align: left; } -button { +button, +a#v_manager.btn.btn-primary { margin-top: 0; margin-bottom: 0; margin-left: 0; - margin-right: 0; + margin-right: 10px; border-width: 2px; border-radius: 10px; @@ -484,7 +485,8 @@ button { box-shadow: var(--shadow); } -button:hover { +button:hover, +a#v_manager.btn.btn-primary { background: var(--color-background-dark); color: var(--color-text); } @@ -492,7 +494,29 @@ button:hover { button#start_all, button#add_streamer { background-color: var(--color-green); - margin-left: 10px; +} + +a#v_manager.btn.btn-primary { + margin-top: 0; + margin-bottom: 0; + margin-left: 0; + margin-right: 0; + + width: auto; + min-width: 106px; + max-width: 106px; + min-height: 22px; + max-height: 22px; + + background-color: var(--color-black); + + + display: inline-block; + +} + +a#v_manager.btn.btn-primary:hover { + background: var(--color-black-dark); } button#start_all:hover, @@ -579,8 +603,8 @@ input#username { div#user-manage { display: grid; - /* grid to the right */ - grid-template-columns: 1fr 1fr; + /* put elements to the left */ + grid-template-columns: 0fr 0fr; grid-template-rows: 1fr 1fr; margin: 0; @@ -589,6 +613,8 @@ div#user-manage { margin-bottom: 10px; height: 40px; text-align: left; + width: auto; + min-width: 400px; font-family: var(--font-jet-brains); } diff --git a/streamonitor/managers/dash_pages/home.py b/streamonitor/managers/dash_pages/home.py new file mode 100644 index 0000000..148ccc9 --- /dev/null +++ b/streamonitor/managers/dash_pages/home.py @@ -0,0 +1,224 @@ +import dash +from dash import Dash +from dash import dcc +from dash import html +from dash import dash_table +from dash import dependencies + +import dash_bootstrap_components as dbc +from streamonitor.managers.dashmanager import lex +from streamonitor.managers.dashmanager import config +from streamonitor.managers.dashmanager import gl_status + +dash.register_page(__name__, + name='StreaMonitor', + path='/', + order=0) + +layout = html.Div([ + html.Div( + [ + html.H1(lex['title']), + html.Div( + [ + html.Button( + lex['stop_all'], + id='stop_all', + n_clicks=0 + ), + html.Button( + lex['start_all'], + id='start_all', + n_clicks=0 + ), + dbc.Button( + lex['v_manager'], + id='v_manager', + href='/video-manager', + n_clicks=0 + ), + ], + id="buttons-all-streamers" + ), + # field to enter username + html.Div(children=[ + dcc.Input( + id="username", + type="text", + placeholder=lex['username'], + value="", + style={ + 'max-height': '36px', + 'height': '36px', + }, + ), + dcc.Dropdown( + id="site", + options=[ + {'label': html.Span(['Chaturbate'], + style={ + 'background-color': + config["color_dropdown_background"]}), + 'value': 'chaturbate'}, + {'label': 'OnlyFans', + 'value': 'onlyfans'}], + className="site-dropdown", + placeholder=lex['site'], + style={ + 'width': '100px', + 'min-height': '30px', + 'height': '40px', + 'margin': 0, + 'padding': 0, + # 'margin-left': '20px', + 'width': '200px', + 'border-radius': '10px', + 'text-align': 'left', + 'font-family': 'font-family: JetBrains Mono, monospace', + 'background-color': config["color_dropdown_background"], + }, + ) + ], + id="user-manage" + ), + + html.Div( + [ + html.Button( + lex['remove'], + id='remove_streamer', + n_clicks=0 + ), + html.Button( + lex['add'], + id='add_streamer', + n_clicks=0 + ), + ], + id="buttons-single-streamer" + ), + + # console output text box + html.Div( + [ + html.Pre( + id='console', + children=lex['console_placeholder'], + ) + ], + id="console_div" + ), + + html.H3('{} . . .'.format( + lex['loading']), id="running"), + html.H3('{} . . .'.format( + lex['loading']), id="cpu_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="ram_usage"), + html.H3('{} . . .'.format( + lex['loading']), id="disk_space"), + + dcc.Interval( + id='interval-component', + # in milliseconds + interval=config["update_interval_s"]*1000, + n_intervals=0 + ), + # sorting fucntionalities + dash_table.DataTable( + id='table', + columns=[{"name": i, "id": i} + for i in gl_status.columns], + data=gl_status.to_dict('records'), + style_cell={ + 'textAlign': 'left', + 'backgroundColor': config["color_cell_background"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + style_data_conditional=[ + { + 'if': { + 'filter_query': '{Status} = "Channel online"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_positive"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Private show"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Not running"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Rate limited"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_warning"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Unknown error"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "Error on downloading"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream for a while"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_neutral"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, + { + 'if': { + 'filter_query': '{Status} = "No stream"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_neutral"], + 'color': config["color_text_inv"], + 'font-family': 'font-family: JetBrains Mono, monospace' + } + ], + style_header={ + 'backgroundColor': config["color_table_header_background"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace', + 'fontWeight': 'bold' + }, + sort_mode="multi", + sort_action="native", + filter_action="native"), + ], + id="main_div"), +]) diff --git a/streamonitor/managers/dash_pages/v_manager.py b/streamonitor/managers/dash_pages/v_manager.py new file mode 100644 index 0000000..f1edebd --- /dev/null +++ b/streamonitor/managers/dash_pages/v_manager.py @@ -0,0 +1,17 @@ +import dash +from dash import Dash +from dash import dcc +from dash import html +from dash import dash_table +from dash import dependencies + + +dash.register_page(__name__, + name='SM - Video Manager', + path='/video-manager', + order=1) + +layout = html.Div([ + html.H1('Video Manager'), + html.Div('This page is not implemented yet.'), +]) diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index ad2c9f4..8699a15 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -3,6 +3,7 @@ from streamonitor.bot import Bot import streamonitor.log as log from streamonitor.manager import Manager +import dash from dash import Dash from dash import dcc from dash import html @@ -10,34 +11,32 @@ import pandas as pd from dash import dash_table from dash import dependencies + from timeit import default_timer as timer import time import shutil import parameters import psutil -import json - - -def load_json(filename: str): - with open(filename, "r") as f: - return json.load(f) - -def load_settings(): - settings = load_json("web-settings.json") - return settings +from streamonitor.managers.jsonmanager import JsonWorker -def load_lexicon(): - lex = load_json("web-lexicon.json") - return lex +# load lexicon +lex = JsonWorker.load("./config/web-lexicon.json") +# load config +config = JsonWorker.load("./config/web-config.json") +# null df +gl_status = pd.DataFrame( + columns=[lex['site'], lex['username'], + lex['started'], lex['status']]) -UPDATE_INTERVAL = 1 # seconds +app = Dash(__name__, title=lex['title'], update_title=None, + external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", + ], + use_pages=True, pages_folder="dash_pages") -# Lexicon for the web interface -lex = load_lexicon() -print(lex) +app.css.config.serve_locally = False class DashManager(Manager): @@ -46,14 +45,6 @@ def __init__(self, streamers): self.logger = log.Logger("manager") def run(self): - app = Dash(__name__, title=lex['title'], update_title=None, - external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", - ]) - - app.css.config.serve_locally = False - - app.title = lex['title'] - def scripts(): pass @@ -67,232 +58,15 @@ def status(): lex['started']: streamer.running, lex['status']: streamer.status()}, ignore_index=True) - return df - def recordings(): - streamer = self.getStreamer( - request.args.get("user"), request.args.get("site")) - try: - temp = [] - for elem \ - in os.listdir("./downloads/{u} [{s}]". - format(u=streamer.username, - s=streamer.siteslug)): - temp.append(elem) - if temp == []: - return "No recordings" - else: - return temp - except: - return "No recordings" + gl_status = df + return df - class colors: - green = '#3D9970' - yellow = '#FFDC00' - red = '#FF4136' - grey = '#DDDDDD' - white = '#EEEEEE' - black = '#191919' + status() app.layout = html.Div( [ - html.Div( - [ - html.H1(lex['title']), - - html.Div( - [ - html.Button( - lex['stop_all'], - id='stop_all', - n_clicks=0 - ), - html.Button( - lex['start_all'], - id='start_all', - n_clicks=0 - ), - ], - id="buttons-all-streamers" - ), - # field to enter username - html.Div(children=[ - dcc.Input( - id="username", - type="text", - placeholder=lex['username'], - value="", - style={ - 'max-height': '36px', - 'height': '36px', - }, - ), - dcc.Dropdown( - id="site", - options=[ - {'label': html.Span(['Chaturbate'], style={ - 'background-color': colors.green}), - 'value': 'chaturbate'}, - {'label': 'OnlyFans', 'value': 'onlyfans'}], - className="site-dropdown", - placeholder=lex['site'], - style={ - 'width': '100px', - 'min-height': '30px', - 'height': '40px', - 'margin': 0, - 'padding': 0, - # 'margin-left': '20px', - 'width': '200px', - 'border-radius': '10px', - 'text-align': 'left', - 'font-family': 'font-family: JetBrains Mono, monospace', - 'background-color': '#606060', - }, - ) - ], - id="user-manage" - ), - - html.Div( - [ - html.Button( - lex['remove'], - id='remove_streamer', - n_clicks=0 - ), - html.Button( - lex['add'], - id='add_streamer', - n_clicks=0 - ), - ], - id="buttons-single-streamer" - ), - - # console output text box - html.Div( - [ - html.Pre( - id='console', - children=lex['console_placeholder'], - ) - ], - id="console_div" - ), - - html.H3('{} . . .'.format( - lex['loading']), id="running"), - html.H3('{} . . .'.format( - lex['loading']), id="cpu_usage"), - html.H3('{} . . .'.format( - lex['loading']), id="ram_usage"), - html.H3('{} . . .'.format( - lex['loading']), id="disk_space"), - - dcc.Interval( - id='interval-component', - interval=UPDATE_INTERVAL*1000, # in milliseconds - n_intervals=0 - ), - # sorting fucntionalities - dash_table.DataTable( - id='table', - columns=[{"name": i, "id": i} - for i in status().columns], - data=status().to_dict('records'), - style_cell={ - 'textAlign': 'left', - 'backgroundColor': '#2f2f2f', - 'color': '#EEEEEE', - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - style_data_conditional=[ - { - 'if': { - 'filter_query': '{Status} = "Channel online"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.green, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Private show"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Not running"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Rate limited"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.yellow, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Unknown error"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.red, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "Error on downloading"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.red, - 'color': colors.white, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "No stream for a while"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.grey, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - }, - { - 'if': { - 'filter_query': '{Status} = "No stream"', - 'column_id': 'Status' - }, - 'backgroundColor': colors.grey, - 'color': colors.black, - 'font-family': 'font-family: JetBrains Mono, monospace' - } - ], - style_header={ - 'backgroundColor': '#101010', - 'color': '#EEEEEE', - 'font-family': 'font-family: JetBrains Mono, monospace', - 'fontWeight': 'bold' - }, - sort_mode="multi", - sort_action="native", - filter_action="native"), - ], - id="main_div"), - + dash.page_container, html.Div( [ html.Div( @@ -309,46 +83,46 @@ class colors: ] ) - # update table - @app.callback( dependencies.Output('table', 'data'), dependencies.Input('interval-component', 'n_intervals')) def update_data(timestamp): + # update table + # status() return status().to_dict('records') - # update running @app.callback( dependencies.Output('running', 'children'), dependencies.Input('interval-component', 'n_intervals')) def update_running(timestamp): + # update running return '=> {}: {}/{}'.\ format(lex['running'], len(status()[status()["Status"] == "Channel online"]), len(status())) - # update cpu usage @app.callback( dependencies.Output('cpu_usage', 'children'), dependencies.Input('interval-component', 'n_intervals')) def update_cpu_usage(timestamp): + # update cpu usage return '=> {}: {}%'.format(lex['cpu'], psutil.cpu_percent()) - # update ram usage @app.callback( dependencies.Output('ram_usage', 'children'), dependencies.Input('interval-component', 'n_intervals')) def update_ram_usage(timestamp): + # update ram usage return '=> {}: {}/{} GB'.\ format(lex['ram'], round(psutil.virtual_memory()[3]/1024/1024/1024, 2), round(psutil.virtual_memory()[0]/1024/1024/1024, 2)) - # update disk space @app.callback( dependencies.Output('disk_space', 'children'), dependencies.Input('interval-component', 'n_intervals')) def update_disk_space(timestamp): + # update disk space return '=> {}: {}/{} {} ({} = {} {})'.\ format(lex['harddrive'], round(shutil.disk_usage("./")[1]/1024/1024/1024, 2), diff --git a/streamonitor/managers/jsonmanager.py b/streamonitor/managers/jsonmanager.py new file mode 100644 index 0000000..9c7326b --- /dev/null +++ b/streamonitor/managers/jsonmanager.py @@ -0,0 +1,17 @@ +import json + + +class JsonWorker: + def load(filepath: str): + ''' + Loads json file from filepath as dict + ''' + with open(filepath, "r") as f: + return json.load(f) + + def save(filepath: str, data: dict): + ''' + Saves data as json file to filepath + ''' + with open(filepath, "w") as f: + json.dump(data, f) diff --git a/web-settings.json b/web-settings.json deleted file mode 100644 index 544b7b4..0000000 --- a/web-settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} \ No newline at end of file From 3fe7b3583066c8ca78a5dbb45cf0f7f357e46760 Mon Sep 17 00:00:00 2001 From: kseen Date: Tue, 29 Aug 2023 23:14:24 +0300 Subject: [PATCH 18/25] Properly working buttons --- config.json | 70 +++++++++++++++++++-- config/web-config.json | 2 +- requirements.txt | 3 +- streamonitor/managers/dash_pages/home.py | 77 +++++++++++++++++++++--- streamonitor/managers/dashmanager.py | 61 +++++++++++++++---- 5 files changed, 187 insertions(+), 26 deletions(-) diff --git a/config.json b/config.json index 93b8fad..8dd246b 100644 --- a/config.json +++ b/config.json @@ -594,11 +594,6 @@ "username": "deeplily", "running": false }, - { - "site": "Chaturbate", - "username": "dose_of_sex", - "running": false - }, { "site": "Chaturbate", "username": "angie_daniels", @@ -633,5 +628,70 @@ "site": "StripChat", "username": "Kitty_Love_cum", "running": false + }, + { + "site": "Chaturbate", + "username": "dose_of_sex", + "running": true + }, + { + "site": "Chaturbate", + "username": "Nicky_spark", + "running": true + }, + { + "site": "Chaturbate", + "username": "newmansteele", + "running": true + }, + { + "site": "Chaturbate", + "username": "mordoreva", + "running": true + }, + { + "site": "Chaturbate", + "username": "minx_girl", + "running": true + }, + { + "site": "Chaturbate", + "username": "megan_world", + "running": true + }, + { + "site": "Chaturbate", + "username": "kittykai_", + "running": true + }, + { + "site": "Chaturbate", + "username": "gwen_joestar", + "running": true + }, + { + "site": "Chaturbate", + "username": "fuckas_de_glace", + "running": true + }, + { + "site": "Chaturbate", + "username": "Evelissa", + "running": true + }, + { + "site": "Chaturbate", + "username": "mabelsherman", + "running": true + }, + { + "site": "Chaturbate", + "username": "byakkomoriko", + "running": true + }, + { + "site": "Chaturbate", + "username": "katemaru", + "running": true } ] \ No newline at end of file diff --git a/config/web-config.json b/config/web-config.json index 6eb2cdd..f40289c 100644 --- a/config/web-config.json +++ b/config/web-config.json @@ -1,5 +1,5 @@ { - "update_interval_s": 5, + "update_interval_s": 1, "color_positive": "#3D9970", "color_warning": "#FFDC00", "color_negative": "#FF4136", diff --git a/requirements.txt b/requirements.txt index d0901f3..cf73f0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ dash>=2.0.0 numpy>=1.24.3 pandas>=2.0.2 vcsi>=7.0.15 -dash_bootstrap_components \ No newline at end of file +dash_bootstrap_components +dash_extensions \ No newline at end of file diff --git a/streamonitor/managers/dash_pages/home.py b/streamonitor/managers/dash_pages/home.py index 148ccc9..10e33b5 100644 --- a/streamonitor/managers/dash_pages/home.py +++ b/streamonitor/managers/dash_pages/home.py @@ -3,7 +3,6 @@ from dash import dcc from dash import html from dash import dash_table -from dash import dependencies import dash_bootstrap_components as dbc from streamonitor.managers.dashmanager import lex @@ -55,13 +54,66 @@ dcc.Dropdown( id="site", options=[ - {'label': html.Span(['Chaturbate'], - style={ - 'background-color': - config["color_dropdown_background"]}), - 'value': 'chaturbate'}, - {'label': 'OnlyFans', - 'value': 'onlyfans'}], + {'label': html.Span(['Amateur.TV'], style={ + 'color': config["color_text_inv"]}), + 'value': 'ATV'}, + + {'label': html.Span(['BongaCams'], style={ + 'color': config["color_text_inv"]}), + 'value': 'BC'}, + + {'label': html.Span(['Cam4'], style={ + 'color': config["color_text_inv"]}), + 'value': 'C4'}, + + {'label': html.Span(['Cams.com'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CC'}, + + {'label': html.Span(['CamSoda'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CS'}, + + {'label': html.Span(['Chaturbate'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CB'}, + + {'label': html.Span(['Cherry.TV'], style={ + 'color': config["color_text_inv"]}), + 'value': 'CHTV'}, + + {'label': html.Span(['Dreamcam VR'], style={ + 'color': config["color_text_inv"]}), + 'value': 'DCVR'}, + + {'label': html.Span(['Flirt4Free'], style={ + 'color': config["color_text_inv"]}), + 'value': 'F4F'}, + + {'label': html.Span(['ManyVids Live'], style={ + 'color': config["color_text_inv"]}), + 'value': 'MV'}, + + {'label': html.Span(['MyFreeCams'], style={ + 'color': config["color_text_inv"]}), + 'value': 'MFC'}, + + {'label': html.Span(['SexChat.hu'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SCHU'}, + + {'label': html.Span(['StreaMate'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SM'}, + + {'label': html.Span(['StripChat'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SC'}, + + {'label': html.Span(['StripChat VR'], style={ + 'color': config["color_text_inv"]}), + 'value': 'SCVR'}, + ], className="site-dropdown", placeholder=lex['site'], style={ @@ -182,6 +234,15 @@ 'color': config["color_text"], 'font-family': 'font-family: JetBrains Mono, monospace' }, + { + 'if': { + 'filter_query': '{Status} = "Nonexistent user"', + 'column_id': 'Status' + }, + 'backgroundColor': config["color_negative"], + 'color': config["color_text"], + 'font-family': 'font-family: JetBrains Mono, monospace' + }, { 'if': { 'filter_query': '{Status} = "Error on downloading"', diff --git a/streamonitor/managers/dashmanager.py b/streamonitor/managers/dashmanager.py index 8699a15..ac3ef8a 100644 --- a/streamonitor/managers/dashmanager.py +++ b/streamonitor/managers/dashmanager.py @@ -3,14 +3,17 @@ from streamonitor.bot import Bot import streamonitor.log as log from streamonitor.manager import Manager -import dash -from dash import Dash -from dash import dcc -from dash import html -import plotly.express as px import pandas as pd -from dash import dash_table -from dash import dependencies +import datetime + +import dash +from dash import Dash, dcc, callback, html, dash_table, dependencies +# from dash import Input +# from dash import Output +# from dash import State + +from dash_extensions.enrich import DashProxy, Input, Output, \ + html, MultiplexerTransform, State from timeit import default_timer as timer import time @@ -31,10 +34,13 @@ lex['started'], lex['status']]) -app = Dash(__name__, title=lex['title'], update_title=None, - external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", - ], - use_pages=True, pages_folder="dash_pages") +app = DashProxy(__name__, + prevent_initial_callbacks=True, + transforms=[MultiplexerTransform()], + title=lex['title'], update_title=None, + external_stylesheets=["https://raw.githubusercontent.com/necolas/normalize.css/master/normalize.css", + ], + use_pages=True, pages_folder="dash_pages") app.css.config.serve_locally = False @@ -134,4 +140,37 @@ def update_disk_space(timestamp): 1024/1024/100, 2), lex['gigabyte']) + @app.callback( + Output('console', 'children'), + Input('stop_all', 'n_clicks')) + def update_stop_all(): + reply = self.execCmd('stop *') + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('start_all', 'n_clicks')) + def update_start_all(): + reply = self.execCmd('start *') + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('remove_streamer', 'n_clicks'), + State('username', 'value')) + def update_remove_streamer(n_clicks, username): + reply = self.execCmd('remove ' + username) + return datetime.datetime.now().__str__() + ' >> ' + reply + + @app.callback( + Output('console', 'children'), + Input('add_streamer', 'n_clicks'), + State('username', 'value'), + State('site', 'value')) + def update_add_streamer(n_clicks, username, site): + reply = self.execCmd('add ' + username + ' ' + site) + return datetime.datetime.now().__str__() + ' >> ' + reply + + # dose_of_sex + app.run(host="127.0.0.1", port=5001) From 721d97a9df30b1b8f1e755eb0354e6c57384a668 Mon Sep 17 00:00:00 2001 From: kseen Date: Sun, 24 Mar 2024 22:14:57 +0300 Subject: [PATCH 19/25] Add TODO --- streamonitor/managers/dash_pages/home.py | 1 + 1 file changed, 1 insertion(+) diff --git a/streamonitor/managers/dash_pages/home.py b/streamonitor/managers/dash_pages/home.py index 10e33b5..5c93da8 100644 --- a/streamonitor/managers/dash_pages/home.py +++ b/streamonitor/managers/dash_pages/home.py @@ -18,6 +18,7 @@ html.Div( [ html.H1(lex['title']), + # TODO: new buttons: delete all downloads, delete all videos html.Div( [ html.Button( From 90dd8bdd130e22195895476114fd5a95205619d3 Mon Sep 17 00:00:00 2001 From: kseen Date: Sun, 24 Mar 2024 22:24:26 +0300 Subject: [PATCH 20/25] Updare reqs --- requirements.txt | 71 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index cf73f0f..2d9de95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,54 @@ -requests>=2.31.0 -terminaltables>=3.1.10 -pyzmq>=25.1.0 -flask>=2.3.2 -termcolor>=2.3.0 -beautifulsoup4>=4.12.2 -websocket-client>=1.5.2 -ffmpy>=0.3.0 -m3u8>=3.5.0 -colorama>=0.4.6 -tqdm>=4.65.0 -dash>=2.0.0 -numpy>=1.24.3 -pandas>=2.0.2 -vcsi>=7.0.15 -dash_bootstrap_components -dash_extensions \ No newline at end of file +backports-datetime-fromisoformat==2.0.1 +beautifulsoup4==4.12.3 +blinker==1.7.0 +cachelib==0.9.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +dash==2.16.1 +dash-bootstrap-components==1.5.0 +dash-core-components==2.0.0 +dash-extensions==1.0.14 +dash-html-components==2.0.0 +dash-table==5.0.0 +dataclass-wizard==0.22.3 +EditorConfig==0.12.4 +ffmpy==0.3.2 +flask==3.0.2 +Flask-Caching==2.1.0 +idna==3.6 +importlib-metadata==7.1.0 +itsdangerous==2.1.2 +Jinja2==3.1.3 +jsbeautifier==1.15.1 +m3u8==4.0.0 +MarkupSafe==2.1.5 +more-itertools==9.1.0 +nest-asyncio==1.6.0 +numpy==1.26.4 +packaging==24.0 +pandas==2.2.1 +parsedatetime==2.6 +pillow==10.2.0 +plotly==5.20.0 +psutil==5.9.8 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyzmq==25.1.2 +requests==2.31.0 +retrying==1.3.4 +six==1.16.0 +soupsieve==2.5 +tenacity==8.2.3 +termcolor==2.4.0 +terminaltables==3.1.10 +texttable==1.7.0 +tqdm==4.66.2 +typing-extensions==4.10.0 +tzdata==2024.1 +urllib3==2.2.1 +vcsi==7.0.16 +websocket-client==1.7.0 +werkzeug==3.0.1 +zipp==3.18.1 From 71ef13bf90f213af63a792dcc96bc0abee5124eb Mon Sep 17 00:00:00 2001 From: kseen Date: Sun, 24 Mar 2024 22:26:52 +0300 Subject: [PATCH 21/25] Update README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 11abe3b..de5b9a2 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,17 @@ The application has the following interfaces: * External console via ZeroMQ (sort of working) * Web interface (only status) +Here the `config.json` file structure: +``` json +[ + { + "site": "StripChat", + "username": "name", + "running": true + }, +] +``` + #### Starting Start the downloader with the following command. Automatically imports all streamers from the config file. From 60d1ce757fc0921129241aba8b2e231aacb9df3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=87=E3=82=A3=E3=83=B3=E3=83=A4?= <94841657+Kseen715@users.noreply.github.com> Date: Mon, 25 Mar 2024 00:11:06 +0300 Subject: [PATCH 22/25] Delete config.json --- config.json | 697 ---------------------------------------------------- 1 file changed, 697 deletions(-) delete mode 100644 config.json diff --git a/config.json b/config.json deleted file mode 100644 index 8dd246b..0000000 --- a/config.json +++ /dev/null @@ -1,697 +0,0 @@ -[ - { - "site": "StripChat", - "username": "raingurl", - "running": false - }, - { - "site": "StripChat", - "username": "Kralya_Smile", - "running": false - }, - { - "site": "StripChat", - "username": "lika_lil", - "running": false - }, - { - "site": "StripChat", - "username": "Misss_Vikk", - "running": false - }, - { - "site": "StripChat", - "username": "Scops_owl", - "running": false - }, - { - "site": "StripChat", - "username": "boobsybootie", - "running": false - }, - { - "site": "StripChat", - "username": "kassablanca_", - "running": false - }, - { - "site": "StripChat", - "username": "littlecarolina", - "running": false - }, - { - "site": "StripChat", - "username": "moon_shy", - "running": false - }, - { - "site": "StripChat", - "username": "Careful_i_bite", - "running": false - }, - { - "site": "StripChat", - "username": "Ginger_Pie", - "running": false - }, - { - "site": "StripChat", - "username": "BUNY_PRINCESS", - "running": false - }, - { - "site": "StripChat", - "username": "kate_pierce", - "running": false - }, - { - "site": "StripChat", - "username": "_Lory_Like_", - "running": false - }, - { - "site": "StripChat", - "username": "Filla--", - "running": false - }, - { - "site": "StripChat", - "username": "susu-dy", - "running": false - }, - { - "site": "StripChat", - "username": "-meowpa-", - "running": false - }, - { - "site": "StripChat", - "username": "MaryBrent", - "running": false - }, - { - "site": "StripChat", - "username": "EroticNana", - "running": false - }, - { - "site": "StripChat", - "username": "FiOnee", - "running": false - }, - { - "site": "StripChat", - "username": "RoseMarceau_", - "running": false - }, - { - "site": "StripChat", - "username": "Lana_Lee_", - "running": false - }, - { - "site": "StripChat", - "username": "Rocks_Babies_", - "running": false - }, - { - "site": "StripChat", - "username": "Kassy_Sun", - "running": false - }, - { - "site": "StripChat", - "username": "Vilgelminaa", - "running": false - }, - { - "site": "StripChat", - "username": "ChloeAttwood_", - "running": false - }, - { - "site": "StripChat", - "username": "DaddysLittleSlutX", - "running": false - }, - { - "site": "StripChat", - "username": "doll_joy", - "running": false - }, - { - "site": "StripChat", - "username": "AkiShina", - "running": false - }, - { - "site": "StripChat", - "username": "Tsumugi_M", - "running": false - }, - { - "site": "StripChat", - "username": "JessAdams", - "running": false - }, - { - "site": "StripChat", - "username": "YourMisano", - "running": false - }, - { - "site": "StripChat", - "username": "TayaMeow", - "running": false - }, - { - "site": "StripChat", - "username": "Eva_Bear", - "running": false - }, - { - "site": "StripChat", - "username": "Badassloli_", - "running": false - }, - { - "site": "StripChat", - "username": "cutiesuee", - "running": false - }, - { - "site": "StripChat", - "username": "amaya_mori", - "running": false - }, - { - "site": "StripChat", - "username": "aerri_lee", - "running": false - }, - { - "site": "StripChat", - "username": "asuna_love", - "running": false - }, - { - "site": "StripChat", - "username": "emma_moanss", - "running": false - }, - { - "site": "BongaCams", - "username": "HollyTailor", - "running": false - }, - { - "site": "StripChat", - "username": "Runa_star", - "running": false - }, - { - "site": "BongaCams", - "username": "Pussy-", - "running": false - }, - { - "site": "BongaCams", - "username": "MiaTonyy", - "running": false - }, - { - "site": "BongaCams", - "username": "PlushCatt", - "running": false - }, - { - "site": "BongaCams", - "username": "milky-cunt", - "running": false - }, - { - "site": "BongaCams", - "username": "WhiteGirl-one", - "running": false - }, - { - "site": "BongaCams", - "username": "Bestblondie", - "running": false - }, - { - "site": "BongaCams", - "username": "-wellcum-", - "running": false - }, - { - "site": "BongaCams", - "username": "MaxineDiaz", - "running": false - }, - { - "site": "BongaCams", - "username": "Bestblondie", - "running": false - }, - { - "site": "Chaturbate", - "username": "donnaayanami", - "running": false - }, - { - "site": "Chaturbate", - "username": "vixenp", - "running": false - }, - { - "site": "Chaturbate", - "username": "alice_kosmos", - "running": false - }, - { - "site": "Chaturbate", - "username": "beka_hernandez", - "running": false - }, - { - "site": "Chaturbate", - "username": "eva_elfyy", - "running": false - }, - { - "site": "Chaturbate", - "username": "francesdonna", - "running": false - }, - { - "site": "Chaturbate", - "username": "aura_69", - "running": false - }, - { - "site": "BongaCams", - "username": "Alisha008", - "running": false - }, - { - "site": "BongaCams", - "username": "Kittenmeew", - "running": false - }, - { - "site": "BongaCams", - "username": "AbsintheGirl", - "running": false - }, - { - "site": "BongaCams", - "username": "MeyShan", - "running": false - }, - { - "site": "BongaCams", - "username": "NeNaSuTnA", - "running": false - }, - { - "site": "BongaCams", - "username": "MarinnaCat", - "running": false - }, - { - "site": "BongaCams", - "username": "KiraLiLime", - "running": false - }, - { - "site": "StripChat", - "username": "SamanthaMorgann", - "running": false - }, - { - "site": "StripChat", - "username": "MadissonSoft", - "running": false - }, - { - "site": "StripChat", - "username": "ummpaLummpa", - "running": false - }, - { - "site": "StripChat", - "username": "wendy_taylor_", - "running": false - }, - { - "site": "BongaCams", - "username": "barsaa7", - "running": false - }, - { - "site": "BongaCams", - "username": "Arisha-w", - "running": false - }, - { - "site": "BongaCams", - "username": "Effy_S", - "running": false - }, - { - "site": "BongaCams", - "username": "SoiaPenka", - "running": false - }, - { - "site": "BongaCams", - "username": "ElizabethGlor", - "running": false - }, - { - "site": "BongaCams", - "username": "Kralya-Smile", - "running": false - }, - { - "site": "BongaCams", - "username": "LilyaLilac", - "running": false - }, - { - "site": "BongaCams", - "username": "cyberannita", - "running": false - }, - { - "site": "BongaCams", - "username": "Baby-Tisha", - "running": false - }, - { - "site": "BongaCams", - "username": "Annelitt", - "running": false - }, - { - "site": "BongaCams", - "username": "AmeliaLean", - "running": false - }, - { - "site": "BongaCams", - "username": "syndicete", - "running": false - }, - { - "site": "BongaCams", - "username": "MiaandMelissa", - "running": false - }, - { - "site": "BongaCams", - "username": "EvaSSly", - "running": false - }, - { - "site": "BongaCams", - "username": "Aru-Koto", - "running": false - }, - { - "site": "BongaCams", - "username": "MariawwEmma", - "running": false - }, - { - "site": "BongaCams", - "username": "LISA-ALYSSA", - "running": false - }, - { - "site": "BongaCams", - "username": "JennaStroker", - "running": false - }, - { - "site": "BongaCams", - "username": "FreyaAdams", - "running": false - }, - { - "site": "BongaCams", - "username": "Lilit1201", - "running": false - }, - { - "site": "Chaturbate", - "username": "fiassquad", - "running": false - }, - { - "site": "Chaturbate", - "username": "ann_lover016", - "running": false - }, - { - "site": "Chaturbate", - "username": "missnicolette1", - "running": false - }, - { - "site": "Chaturbate", - "username": "hornybunnys", - "running": false - }, - { - "site": "Chaturbate", - "username": "lollypetitte_", - "running": false - }, - { - "site": "Chaturbate", - "username": "fairyinthewild", - "running": false - }, - { - "site": "Chaturbate", - "username": "pornxxxcouple", - "running": false - }, - { - "site": "Chaturbate", - "username": "sweetpornface", - "running": false - }, - { - "site": "Chaturbate", - "username": "santiago_and_lana", - "running": false - }, - { - "site": "Chaturbate", - "username": "angelandfairieskink", - "running": false - }, - { - "site": "BongaCams", - "username": "Alastaira", - "running": false - }, - { - "site": "BongaCams", - "username": "Vasilisa-V", - "running": false - }, - { - "site": "BongaCams", - "username": "ElisBraun", - "running": false - }, - { - "site": "BongaCams", - "username": "Diafoxy", - "running": false - }, - { - "site": "BongaCams", - "username": "Lola3000", - "running": false - }, - { - "site": "BongaCams", - "username": "CandyKushhh", - "running": false - }, - { - "site": "BongaCams", - "username": "KateDanverr", - "running": false - }, - { - "site": "BongaCams", - "username": "crispy-tati-", - "running": false - }, - { - "site": "BongaCams", - "username": "MissCatty", - "running": false - }, - { - "site": "BongaCams", - "username": "HotDemonic", - "running": false - }, - { - "site": "Chaturbate", - "username": "kyrrax23", - "running": false - }, - { - "site": "Chaturbate", - "username": "emmabraun", - "running": false - }, - { - "site": "Chaturbate", - "username": "_paprika", - "running": false - }, - { - "site": "Chaturbate", - "username": "_b_a_n_s_h_e_e_", - "running": false - }, - { - "site": "Chaturbate", - "username": "marcussanchez", - "running": false - }, - { - "site": "Chaturbate", - "username": "yourlovelyjul", - "running": false - }, - { - "site": "Chaturbate", - "username": "mollyflwers", - "running": false - }, - { - "site": "Chaturbate", - "username": "your_cute_marshmallow", - "running": false - }, - { - "site": "Chaturbate", - "username": "kotosqvad", - "running": false - }, - { - "site": "Chaturbate", - "username": "deeplily", - "running": false - }, - { - "site": "Chaturbate", - "username": "angie_daniels", - "running": false - }, - { - "site": "Chaturbate", - "username": "frori2", - "running": false - }, - { - "site": "Chaturbate", - "username": "lolliruth", - "running": false - }, - { - "site": "Chaturbate", - "username": "projektmelody", - "running": false - }, - { - "site": "Chaturbate", - "username": "alexisward", - "running": false - }, - { - "site": "Chaturbate", - "username": "limy_fresh", - "running": false - }, - { - "site": "StripChat", - "username": "Kitty_Love_cum", - "running": false - }, - { - "site": "Chaturbate", - "username": "dose_of_sex", - "running": true - }, - { - "site": "Chaturbate", - "username": "Nicky_spark", - "running": true - }, - { - "site": "Chaturbate", - "username": "newmansteele", - "running": true - }, - { - "site": "Chaturbate", - "username": "mordoreva", - "running": true - }, - { - "site": "Chaturbate", - "username": "minx_girl", - "running": true - }, - { - "site": "Chaturbate", - "username": "megan_world", - "running": true - }, - { - "site": "Chaturbate", - "username": "kittykai_", - "running": true - }, - { - "site": "Chaturbate", - "username": "gwen_joestar", - "running": true - }, - { - "site": "Chaturbate", - "username": "fuckas_de_glace", - "running": true - }, - { - "site": "Chaturbate", - "username": "Evelissa", - "running": true - }, - { - "site": "Chaturbate", - "username": "mabelsherman", - "running": true - }, - { - "site": "Chaturbate", - "username": "byakkomoriko", - "running": true - }, - { - "site": "Chaturbate", - "username": "katemaru", - "running": true - } -] \ No newline at end of file From d1eb45e24fe7ed978efa67ae6abb0439fff48de9 Mon Sep 17 00:00:00 2001 From: kseen Date: Mon, 25 Mar 2024 00:13:13 +0300 Subject: [PATCH 23/25] Update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 93cc4e3..a4c6728 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,6 @@ ENV/ # CUSTOM .idea/ -config.json videos/* downloads/* +config.json From c2c31b6cf601692df83856aa6479630742bb0b3e Mon Sep 17 00:00:00 2001 From: kseen Date: Mon, 25 Mar 2024 23:47:53 +0300 Subject: [PATCH 24/25] Add notes for future dev --- mac.py | 1 + make_vcsis.py | 1 + 2 files changed, 2 insertions(+) diff --git a/mac.py b/mac.py index fda8d0c..fd9e5cd 100644 --- a/mac.py +++ b/mac.py @@ -49,6 +49,7 @@ def call_function(filename): threads = [] for f in tqdm(files, desc=f'{col.Style.RESET_ALL}[MAC]: Starting threads')\ if tqdm_opt else files: + # TODO: limit thread count to a CPU count t = th.Thread(target=call_function, args=(f,)) threads.append(t) t.start() diff --git a/make_vcsis.py b/make_vcsis.py index 0c151a6..b0a0a9f 100644 --- a/make_vcsis.py +++ b/make_vcsis.py @@ -5,6 +5,7 @@ def main(filename: str = 'videos', overwrite: bool = False): + # TODO: reduce the grid size to 5x5 (?) args = ["vcsi", "-t", "-w 7680", "-g 8x8"] if not overwrite: args.append("--no-overwrite") From e302edc3b1cc50c32e49a9e79af0e3068510e25f Mon Sep 17 00:00:00 2001 From: kseen Date: Wed, 17 Apr 2024 02:22:30 +0300 Subject: [PATCH 25/25] Reduce vcsi size from 8x8 to 5x5 --- make_vcsis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/make_vcsis.py b/make_vcsis.py index b0a0a9f..80dd8cc 100644 --- a/make_vcsis.py +++ b/make_vcsis.py @@ -5,8 +5,7 @@ def main(filename: str = 'videos', overwrite: bool = False): - # TODO: reduce the grid size to 5x5 (?) - args = ["vcsi", "-t", "-w 7680", "-g 8x8"] + args = ["vcsi", "-t", "-w 7680", "-g 5x5"] if not overwrite: args.append("--no-overwrite") args.append(filename)