From 2c2e4c2e2405210fc0bdb63f6a234fce5e8ed169 Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 14:57:05 -0400 Subject: [PATCH 01/12] chore: simplify + optimize --- twitchtube/clips.py | 79 ++++++++++--------- twitchtube/logging.py | 15 ++-- twitchtube/utils.py | 42 +++++------ twitchtube/video.py | 172 ++++++++++++++++++++---------------------- 4 files changed, 149 insertions(+), 159 deletions(-) diff --git a/twitchtube/clips.py b/twitchtube/clips.py index 7d08c19..ed0aafe 100644 --- a/twitchtube/clips.py +++ b/twitchtube/clips.py @@ -1,14 +1,10 @@ import datetime -from math import ceil -from json import dump -import urllib.request import re +import urllib.request -from .logging import Log -from .utils import format_blacklist, is_blacklisted from .api import get - -log = Log() +from .logging import Log as log +from .utils import format_blacklist, is_blacklisted def get_data(slug: str, oauth_token: str, client_id: str) -> dict: @@ -83,35 +79,31 @@ def download_clip(clip: str, basepath: str, oauth_token: str, client_id: str) -> def get_clips( - blacklist: list, - category: str, - id_: str, - name: str, - path: str, - seconds: float, - ids: list, - client_id: str, - oauth_token: str, - period: int, - language: str, - limit: int, + blacklist: list, + category: str, + id_: str, + name: str, + path: str, + seconds: float, + ids: list, + client_id: str, + oauth_token: str, + period: int, + language: str, + limit: int, ) -> (dict, list, list): """ Gets the top clips for given game, returns JSON response from the Helix API endpoint. """ - data = {} - new_ids = [] - new_titles = [] - headers = {"Accept": "application/vnd.twitchtv.v5+json", "Client-ID": client_id} # params = {"period": period, "limit": limit} params = { "ended_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "started_at": ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(hours=period) + datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(hours=period) ).isoformat(), "first": limit, } @@ -129,17 +121,18 @@ def get_clips( if response.get("error") == "Internal Server Error": # the error is twitch's fault, we try again get_clips( - blacklist, - category, - name, - path, - seconds, - ids, - client_id, - oauth_token, - period, - language, - limit, + blacklist=blacklist, + category=category, + id_=id_, + name=name, + path=path, + seconds=seconds, + ids=ids, + client_id=client_id, + oauth_token=oauth_token, + period=period, + language=language, + limit=limit, ) else: @@ -150,6 +143,10 @@ def get_clips( formatted_blacklist = format_blacklist(blacklist, oauth_token, client_id) if "data" in response: + data = {} + new_ids = [] + new_titles = [] + for clip in response["data"]: clip_id = clip["id"] duration = clip["duration"] @@ -158,9 +155,9 @@ def get_clips( break if ( - clip_id not in ids - and not is_blacklisted(clip, formatted_blacklist) - and (language == clip["language"] or not language) + clip_id not in ids + and not is_blacklisted(clip, formatted_blacklist) + and (language == clip["language"] or not language) ): data[clip["id"]] = { "url": clip["url"], @@ -183,8 +180,8 @@ def download_clips(data: dict, path: str, oauth_token: str, client_id: str) -> l """ names = [] - for clip in data: - download_clip(data[clip]["url"], path, oauth_token, client_id) + for clip, value in data.items(): + download_clip(value["url"], path, oauth_token, client_id) name = data[clip]["display_name"] diff --git a/twitchtube/logging.py b/twitchtube/logging.py index 1146a60..0a8bbd8 100644 --- a/twitchtube/logging.py +++ b/twitchtube/logging.py @@ -33,17 +33,22 @@ def log(color: int, sort: str, text: str) -> None: class Log: - def info(self, text: str): + @staticmethod + def info(text: str): log(f.GREEN, "info", text) - def error(self, text: str): + @staticmethod + def error(text: str): log(f.RED, "error", text) - def warn(self, text: str): + @staticmethod + def warn(text: str): log(f.YELLOW, "warn", text) - def clip(self, text: str): + @staticmethod + def clip(text: str): log(f.CYAN, "clip", text) - def debug(self, text: str): + @staticmethod + def debug(text: str): log(f.BLUE, "debug", text) diff --git a/twitchtube/utils.py b/twitchtube/utils.py index 433dec1..79c5214 100644 --- a/twitchtube/utils.py +++ b/twitchtube/utils.py @@ -1,12 +1,12 @@ from datetime import date -from string import ascii_lowercase, digits from random import choice +from string import ascii_lowercase, digits + +from requests import get as rget from .api import get -from .exceptions import InvalidCategory from .config import CLIP_PATH - -from requests import get as rget +from .exceptions import InvalidCategory def get_date() -> str: @@ -34,19 +34,19 @@ def get_current_version(project: str) -> str: response = rget( f"https://raw.githubusercontent.com/offish/{project}/master/{project}/__init__.py" ).text - response = response[response.index(txt) :].replace(txt, "") + response = response[response.index(txt):].replace(txt, "") return response[: response.index('"\n')].replace('"', "") def create_video_config( - path: str, - file_name: str, - title: str, - description: str, - thumbnail: str, - tags: list, - names: list, + path: str, + file_name: str, + title: str, + description: str, + thumbnail: str, + tags: list, + names: list, ) -> dict: return { "file": f"{path}/{file_name}.mp4", @@ -58,10 +58,10 @@ def create_video_config( def get_category(category: str) -> str: - if category == "g" or category == "game": + if category in {"g", "game"}: return "game" - if category == "c" or category == "channel": + if category in {"c", "channel"}: return "channel" raise InvalidCategory( @@ -135,14 +135,10 @@ def format_blacklist(blacklist: list, oauth_token: str, client_id: str) -> list: def is_blacklisted(clip: dict, blacklist: list) -> bool: - if "broadcaster_id" in clip: - if "channel " + clip["broadcaster_id"].lower() in [ - i.lower() for i in blacklist - ]: - return True - - if clip.get("game_id"): - if "game " + clip["game_id"] in blacklist: - return True + if "broadcaster_id" in clip and "channel " + clip["broadcaster_id"].lower() in [i.lower() for i in blacklist]: + return True + + if clip.get("game_id") and "game " + clip["game_id"] in blacklist: + return True return False diff --git a/twitchtube/video.py b/twitchtube/video.py index ce382eb..0c5114d 100644 --- a/twitchtube/video.py +++ b/twitchtube/video.py @@ -1,80 +1,78 @@ -from pathlib import Path -from json import dump -from glob import glob import os - -from twitchtube import __version__ as twitchtube_version -from .exceptions import * -from .logging import Log -from .config import * -from .utils import * -from .clips import get_clips, download_clips +from glob import glob +from json import dump +from pathlib import Path from moviepy.editor import VideoFileClip, concatenate_videoclips from opplast import Upload, __version__ as opplast_version -log = Log() +from twitchtube import __version__ as twitchtube_version +from .clips import get_clips, download_clips +from .config import * +from .exceptions import * +from .logging import Log as log +from .utils import * # add language as param def make_video( - # required - data: list = DATA, - blacklist: list = BLACKLIST, - # other - path: str = get_path(), - check_version: bool = CHECK_VERSION, - # twitch - client_id: str = CLIENT_ID, - oauth_token: str = OAUTH_TOKEN, - period: int = PERIOD, - language: str = LANGUAGE, - limit: int = LIMIT, - # selenium - profile_path: str = ROOT_PROFILE_PATH, - executable_path: str = EXECUTABLE_PATH, - sleep: int = SLEEP, - headless: bool = HEADLESS, - debug: bool = DEBUG, - # video options - render_video: bool = RENDER_VIDEO, - file_name: str = FILE_NAME, - resolution: tuple = RESOLUTION, - frames: int = FRAMES, - video_length: float = VIDEO_LENGTH, - resize_clips: bool = RESIZE_CLIPS, - enable_intro: bool = ENABLE_INTRO, - resize_intro: bool = RESIZE_INTRO, - intro_path: str = INTRO_FILE_PATH, - enable_transition: bool = ENABLE_TRANSITION, - resize_transition: bool = RESIZE_TRANSITION, - transition_path: str = TRANSITION_FILE_PATH, - enable_outro: bool = ENABLE_OUTRO, - resize_outro: bool = RESIZE_OUTRO, - outro_path: str = OUTRO_FILE_PATH, - # other options - save_file: bool = SAVE_TO_FILE, - save_file_name: str = SAVE_FILE_NAME, - upload_video: bool = UPLOAD_TO_YOUTUBE, - delete_clips: bool = DELETE_CLIPS, - # youtube - title: str = TITLE, - description: str = DESCRIPTION, - thumbnail: str = THUMBNAIL, - tags: list = TAGS, + # required + data: list = DATA, + blacklist: list = BLACKLIST, + # other + path: str = get_path(), + check_version: bool = CHECK_VERSION, + # twitch + client_id: str = CLIENT_ID, + oauth_token: str = OAUTH_TOKEN, + period: int = PERIOD, + language: str = LANGUAGE, + limit: int = LIMIT, + # selenium + profile_path: str = ROOT_PROFILE_PATH, + executable_path: str = EXECUTABLE_PATH, + sleep: int = SLEEP, + headless: bool = HEADLESS, + debug: bool = DEBUG, + # video options + render_video: bool = RENDER_VIDEO, + file_name: str = FILE_NAME, + resolution: tuple = RESOLUTION, + frames: int = FRAMES, + video_length: float = VIDEO_LENGTH, + resize_clips: bool = RESIZE_CLIPS, + enable_intro: bool = ENABLE_INTRO, + resize_intro: bool = RESIZE_INTRO, + intro_path: str = INTRO_FILE_PATH, + enable_transition: bool = ENABLE_TRANSITION, + resize_transition: bool = RESIZE_TRANSITION, + transition_path: str = TRANSITION_FILE_PATH, + enable_outro: bool = ENABLE_OUTRO, + resize_outro: bool = RESIZE_OUTRO, + outro_path: str = OUTRO_FILE_PATH, + # other options + save_file: bool = SAVE_TO_FILE, + save_file_name: str = SAVE_FILE_NAME, + upload_video: bool = UPLOAD_TO_YOUTUBE, + delete_clips: bool = DELETE_CLIPS, + # youtube + title: str = TITLE, + description: str = DESCRIPTION, + thumbnail: str = THUMBNAIL, + tags: list = TAGS, ) -> None: if check_version: try: for project, version in zip( - [ - "twitchtube", - "opplast", - ], - [ - twitchtube_version, - opplast_version, - ], + [ + "twitchtube", + "opplast", + ], + [ + twitchtube_version, + opplast_version, + ], ): current = get_current_version(project) @@ -214,7 +212,7 @@ def make_video( files = glob(f"{path}/*.mp4") for file in files: - if not file.replace("\\", "/") == path + f"/{file_name}.mp4": + if file.replace("\\", "/") != path + f"/{file_name}.mp4": try: os.remove(file) log.clip(f"Deleted {file.replace(path, '')}") @@ -240,20 +238,20 @@ def add_clip(path: str, resolution: tuple, resize: bool = True) -> VideoFileClip def render( - path: str, - file_name: str, - resolution: tuple, - frames: int, - resize_clips: bool, - enable_intro: bool, - resize_intro: bool, - intro_path: str, - enable_transition: bool, - resize_transition: bool, - transition_path: str, - enable_outro: bool, - resize_outro: bool, - outro_path: str, + path: str, + file_name: str, + resolution: tuple, + frames: int, + resize_clips: bool, + enable_intro: bool, + resize_intro: bool, + intro_path: str, + enable_transition: bool, + resize_transition: bool, + transition_path: str, + enable_outro: bool, + resize_outro: bool, + outro_path: str, ) -> None: """ Concatenates a video with given path. @@ -265,34 +263,28 @@ def render( video = [] if enable_intro: - video.append(add_clip(intro_path, resolution, resize_intro == True)) - - number = 0 + video.append(add_clip(intro_path, resolution, resize_intro)) clips = get_clip_paths(path) - for clip in clips: + for number, clip in enumerate(clips): # Don't add transition if it's the first or last clip - if enable_transition and not (number == 0 or number == len(clips)): - video.append( - add_clip(transition_path, resolution, resize_transition == True) - ) + if enable_transition and number not in [0, len(clips)]: + video.append(add_clip(transition_path, resolution, resize_transition)) - video.append(add_clip(clip, resolution, resize_clips == True)) + video.append(add_clip(clip, resolution, resize_clips)) # Just so we get cleaner logging name = clip.replace(path, "").replace("_", " ").replace("\\", "") log.info(f"Added {name} to be rendered") - number += 1 - del clip del name if enable_outro: - video.append(add_clip(outro_path, resolution, resize_outro == True)) + video.append(add_clip(outro_path, resolution, resize_outro)) final = concatenate_videoclips(video, method="compose") final.write_videofile( From da02f8176e582b60845fb40529b23ad4ac7d68fb Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 15:36:32 -0400 Subject: [PATCH 02/12] chore: consistent requests imports --- twitchtube/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twitchtube/utils.py b/twitchtube/utils.py index 79c5214..3f95cf0 100644 --- a/twitchtube/utils.py +++ b/twitchtube/utils.py @@ -2,7 +2,7 @@ from random import choice from string import ascii_lowercase, digits -from requests import get as rget +import requests from .api import get from .config import CLIP_PATH @@ -31,7 +31,7 @@ def get_description(description: str, names: list) -> str: def get_current_version(project: str) -> str: txt = '__version__ = "' - response = rget( + response = requests.get( f"https://raw.githubusercontent.com/offish/{project}/master/{project}/__init__.py" ).text response = response[response.index(txt):].replace(txt, "") From e8a5ca8bc7a5ead2116b8203f966523fe91b3c9a Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 15:36:53 -0400 Subject: [PATCH 03/12] chore: add requests to requirements.txt --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c55c85f..734e8c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ moviepy==1.0.3 -colorama +colorama~=0.4.4 selenium -opplast +opplast~=1.0.7 +requests~=2.26.0 \ No newline at end of file From 4882d8f5dbdcaafc8538c259f0db160733cbed76 Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 15:44:08 -0400 Subject: [PATCH 04/12] chore: typos & unused import --- twitchtube/api.py | 2 -- twitchtube/clips.py | 2 +- twitchtube/config.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/twitchtube/api.py b/twitchtube/api.py index 1c04677..0d2808f 100644 --- a/twitchtube/api.py +++ b/twitchtube/api.py @@ -1,5 +1,3 @@ -import json - import requests local = locals() diff --git a/twitchtube/clips.py b/twitchtube/clips.py index ed0aafe..283ab5e 100644 --- a/twitchtube/clips.py +++ b/twitchtube/clips.py @@ -10,7 +10,7 @@ def get_data(slug: str, oauth_token: str, client_id: str) -> dict: """ Gets the data from a given slug, - returns a JSON respone from the Helix API endpoint + returns a JSON response from the Helix API endpoint """ response = get("data", slug=slug, oauth_token=oauth_token, client_id=client_id) diff --git a/twitchtube/config.py b/twitchtube/config.py index aa0e27b..c16b0da 100644 --- a/twitchtube/config.py +++ b/twitchtube/config.py @@ -36,9 +36,9 @@ RESOLUTION = ( 720, 1280, -) # Resoultion of the rendered video (height, width) for 1080p: ((1080, 1920)) +) # Resolution of the rendered video (height, width) for 1080p: ((1080, 1920)) FRAMES = 30 # Frames per second (30/60) -VIDEO_LENGTH = 10.5 # Minumum video length in minutes (doesn't always work) +VIDEO_LENGTH = 10.5 # Minimum video length in minutes (doesn't always work) RESIZE_CLIPS = True # Resize clips to fit RESOLUTION (True/False) If any RESIZE option is set to False the video might end up having a weird resolution FILE_NAME = "rendered" # Name of the rendered video ENABLE_INTRO = False # Enable (True/False) From 968af8b1f70439d6ffaddfb519b1124c65b0772b Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 15:45:52 -0400 Subject: [PATCH 05/12] feat: `TwitchTubeError` base exception class --- twitchtube/exceptions.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/twitchtube/exceptions.py b/twitchtube/exceptions.py index 0c28f80..1885c42 100644 --- a/twitchtube/exceptions.py +++ b/twitchtube/exceptions.py @@ -1,10 +1,14 @@ -class InvalidCategory(Exception): - pass +class TwitchTubeError(Exception): + """ General error class for TwitchTube.""" -class VideoPathAlreadyExists(Exception): - pass +class InvalidCategory(TwitchTubeError): + """ Error for when the specified category is invalid """ -class NoClipsFound(Exception): - pass +class VideoPathAlreadyExists(TwitchTubeError): + """ Error for when a path already exists. """ + + +class NoClipsFound(TwitchTubeError): + """ Error for when no clips are found. """ From 3ae2aa0a6b50021fa945ebdc1f78080467f20380 Mon Sep 17 00:00:00 2001 From: Hexiro <42787085+Hexiro@users.noreply.github.com> Date: Sun, 19 Sep 2021 16:19:02 -0400 Subject: [PATCH 06/12] chore: .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..64e2c5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv +.idea +clips +*.pyc \ No newline at end of file From 097b50f12ace021700cca018dfeedf4f56d8a9b2 Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:04:33 +0200 Subject: [PATCH 07/12] add 0 for all time top clips #78 --- twitchtube/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twitchtube/config.py b/twitchtube/config.py index c16b0da..0a586f5 100644 --- a/twitchtube/config.py +++ b/twitchtube/config.py @@ -19,7 +19,7 @@ # twitch CLIENT_ID = "" # Twitch Client ID OAUTH_TOKEN = "" # Twitch OAuth Token -PERIOD = 24 # how many hours since the clip's creation should've passed e.g. 24, 48 etc +PERIOD = 24 # how many hours since the clip's creation should've passed e.g. 24, 48 etc 0 for all time LANGUAGE = "en" # en, es, th etc. LIMIT = 100 # 1-100 From cd35da5e03df3378dbc10a6abadc7de856127b18 Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:04:49 +0200 Subject: [PATCH 08/12] update version --- twitchtube/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twitchtube/__init__.py b/twitchtube/__init__.py index 99b5633..d635c38 100644 --- a/twitchtube/__init__.py +++ b/twitchtube/__init__.py @@ -1,4 +1,4 @@ __title__ = "twitchtube" __author__ = "offish" __license__ = "MIT" -__version__ = "1.6.5" +__version__ = "1.6.6" From 887780d31412c93aa68e324dd5ce8c70a19f0827 Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:05:14 +0200 Subject: [PATCH 09/12] merge user and game into helix --- twitchtube/api.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/twitchtube/api.py b/twitchtube/api.py index 0d2808f..36439c8 100644 --- a/twitchtube/api.py +++ b/twitchtube/api.py @@ -17,23 +17,13 @@ def data(slug: str, oauth_token: str, client_id: str) -> requests.Response: ) -def game(game_list: list, oauth_token: str, client_id: str) -> requests.Response: - # returns data about every specified name of the game (including it's id) - # e.g. [Minecraft] -> {'id': '27471', 'name': 'Minecraft', - # 'box_art_url': 'https://static-cdn.jtvnw.net/ttv-boxart/Minecraft-{width}x{height}.jpg'} +def helix( + category: str, data: list, oauth_token: str, client_id: str +) -> requests.Response: return request( - "helix/games", + "helix/" + category, {"Authorization": "Bearer " + oauth_token, "Client-Id": client_id}, - {"name": game_list} - ) - - -def user(user_list: list, oauth_token: str, client_id: str) -> requests.Response: - # just like game() but for users - return request( - "helix/users", - {"Authorization": "Bearer " + oauth_token, "Client-Id": client_id}, - {"login": user_list} + {"login" if category == "users" else "name": data}, ) From 05fa21636e76a3533c492ee1abff90850e6c1137 Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:06:21 +0200 Subject: [PATCH 10/12] add all time filter and black format --- twitchtube/clips.py | 57 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/twitchtube/clips.py b/twitchtube/clips.py index 283ab5e..353594d 100644 --- a/twitchtube/clips.py +++ b/twitchtube/clips.py @@ -79,18 +79,18 @@ def download_clip(clip: str, basepath: str, oauth_token: str, client_id: str) -> def get_clips( - blacklist: list, - category: str, - id_: str, - name: str, - path: str, - seconds: float, - ids: list, - client_id: str, - oauth_token: str, - period: int, - language: str, - limit: int, + blacklist: list, + category: str, + id_: str, + name: str, + path: str, + seconds: float, + ids: list, + client_id: str, + oauth_token: str, + period: int, + language: str, + limit: int, ) -> (dict, list, list): """ Gets the top clips for given game, returns JSON response @@ -100,18 +100,20 @@ def get_clips( # params = {"period": period, "limit": limit} params = { - "ended_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "started_at": ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(hours=period) - ).isoformat(), "first": limit, } - if category == "channel": - params["broadcaster_id"] = id_ - else: - params["game_id"] = id_ + if period: + params = { + **params, + "ended_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "started_at": ( + datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(hours=period) + ).isoformat(), + } + + params["broadcaster_id" if category == "channel" else "game_id"] = id_ log.info(f"Getting clips for {category} {name}") @@ -155,9 +157,9 @@ def get_clips( break if ( - clip_id not in ids - and not is_blacklisted(clip, formatted_blacklist) - and (language == clip["language"] or not language) + clip_id not in ids + and not is_blacklisted(clip, formatted_blacklist) + and (language == clip["language"] or not language) ): data[clip["id"]] = { "url": clip["url"], @@ -182,10 +184,7 @@ def download_clips(data: dict, path: str, oauth_token: str, client_id: str) -> l for clip, value in data.items(): download_clip(value["url"], path, oauth_token, client_id) + names.append(data[clip]["display_name"]) - name = data[clip]["display_name"] - - names.append(name) - - log.info(f"Downloaded {len(data)} clips from this batch.\n") + log.info(f"Downloaded {len(data)} clips from this batch\n") return names From 590e4185585fa736e5f4081ad65b4a9c5a78991b Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:07:31 +0200 Subject: [PATCH 11/12] rename convert_name_to_ids and shorten code --- twitchtube/utils.py | 108 ++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 60 deletions(-) diff --git a/twitchtube/utils.py b/twitchtube/utils.py index 3f95cf0..22126e9 100644 --- a/twitchtube/utils.py +++ b/twitchtube/utils.py @@ -24,9 +24,7 @@ def get_path() -> str: def get_description(description: str, names: list) -> str: - for name in names: - description += f"https://twitch.tv/{name}\n" - return description + return description + "".join([f"https://twitch.tv/{name}\n" for name in names]) def get_current_version(project: str) -> str: @@ -34,19 +32,19 @@ def get_current_version(project: str) -> str: response = requests.get( f"https://raw.githubusercontent.com/offish/{project}/master/{project}/__init__.py" ).text - response = response[response.index(txt):].replace(txt, "") + response = response[response.index(txt) :].replace(txt, "") return response[: response.index('"\n')].replace('"', "") def create_video_config( - path: str, - file_name: str, - title: str, - description: str, - thumbnail: str, - tags: list, - names: list, + path: str, + file_name: str, + title: str, + description: str, + thumbnail: str, + tags: list, + names: list, ) -> dict: return { "file": f"{path}/{file_name}.mp4", @@ -58,15 +56,12 @@ def create_video_config( def get_category(category: str) -> str: - if category in {"g", "game"}: - return "game" + if category not in ["g", "game", "c", "channel"]: + raise InvalidCategory( + category + ' is not supported. Use "g", "game", "c" or "channel"' + ) - if category in {"c", "channel"}: - return "channel" - - raise InvalidCategory( - category + ' is not supported. Use "g", "game", "c" or "channel"' - ) + return "game" if category in ["g", "game"] else "channel" def get_category_and_name(entry: str) -> (str, str): @@ -76,38 +71,36 @@ def get_category_and_name(entry: str) -> (str, str): return category, name -def convert_name_to_ids(data: list, oauth_token: str, client_id: str) -> list: - # all data that is gets - new_data = [] - users_to_check, games_to_check = [], [] - user_info, game_info = [], [] - - for entry in data: - category, name = get_category_and_name(entry) - if category == "channel": - users_to_check.append(name) - elif category == "game": - games_to_check.append(name) - - # if there are more than 100 entries in users_to_check or games_to_check, this *WILL NOT WORK* - if users_to_check: - user_info = get( - "user", - user_list=users_to_check, - oauth_token=oauth_token, - client_id=client_id, - )["data"] - if games_to_check: - game_info = get( - "game", - game_list=games_to_check, - oauth_token=oauth_token, - client_id=client_id, - )["data"] - - return [("channel", i["id"], i["display_name"]) for i in user_info] + [ - ("game", i["id"], i["name"]) for i in game_info - ] +def name_to_ids(data: list, oauth_token: str, client_id: str) -> list: + result = [] + + for category, helix_category, helix_name in [ + (["channel", "c"], "users", "display_name"), + (["game", "g"], "games", "name"), + ]: + current_list = [] + + for entry in data: + c, n = get_category_and_name(entry) + + if c in category: + current_list.append(n) + + if len(current_list) > 0: + info = ( + get( + "helix", + category=helix_category, + data=current_list, + oauth_token=oauth_token, + client_id=client_id, + ).get("data") + or [] + ) + + result += [(category[0], i["id"], i[helix_name]) for i in info] + + return result def remove_blacklisted(data: list, blacklist: list) -> (bool, list): @@ -130,15 +123,10 @@ def remove_blacklisted(data: list, blacklist: list) -> (bool, list): def format_blacklist(blacklist: list, oauth_token: str, client_id: str) -> list: - formatted = convert_name_to_ids(blacklist, oauth_token, client_id) - return [f"{i[0]} {i[1]}" for i in formatted] + return [f"{i[0]} {i[1]}" for i in name_to_ids(blacklist, oauth_token, client_id)] def is_blacklisted(clip: dict, blacklist: list) -> bool: - if "broadcaster_id" in clip and "channel " + clip["broadcaster_id"].lower() in [i.lower() for i in blacklist]: - return True - - if clip.get("game_id") and "game " + clip["game_id"] in blacklist: - return True - - return False + return ( + "broadcaster_id" in clip and "channel " + clip["broadcaster_id"] in blacklist + ) or ("game_id" in clip and "game " + clip["game_id"] in blacklist) From 0101dd13fde6f6ad9ce88acedc26310bbf27c6af Mon Sep 17 00:00:00 2001 From: offish Date: Tue, 21 Sep 2021 21:07:55 +0200 Subject: [PATCH 12/12] black format and rename convert_name_to_ids --- twitchtube/video.py | 138 ++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/twitchtube/video.py b/twitchtube/video.py index 0c5114d..87628ed 100644 --- a/twitchtube/video.py +++ b/twitchtube/video.py @@ -16,63 +16,63 @@ # add language as param def make_video( - # required - data: list = DATA, - blacklist: list = BLACKLIST, - # other - path: str = get_path(), - check_version: bool = CHECK_VERSION, - # twitch - client_id: str = CLIENT_ID, - oauth_token: str = OAUTH_TOKEN, - period: int = PERIOD, - language: str = LANGUAGE, - limit: int = LIMIT, - # selenium - profile_path: str = ROOT_PROFILE_PATH, - executable_path: str = EXECUTABLE_PATH, - sleep: int = SLEEP, - headless: bool = HEADLESS, - debug: bool = DEBUG, - # video options - render_video: bool = RENDER_VIDEO, - file_name: str = FILE_NAME, - resolution: tuple = RESOLUTION, - frames: int = FRAMES, - video_length: float = VIDEO_LENGTH, - resize_clips: bool = RESIZE_CLIPS, - enable_intro: bool = ENABLE_INTRO, - resize_intro: bool = RESIZE_INTRO, - intro_path: str = INTRO_FILE_PATH, - enable_transition: bool = ENABLE_TRANSITION, - resize_transition: bool = RESIZE_TRANSITION, - transition_path: str = TRANSITION_FILE_PATH, - enable_outro: bool = ENABLE_OUTRO, - resize_outro: bool = RESIZE_OUTRO, - outro_path: str = OUTRO_FILE_PATH, - # other options - save_file: bool = SAVE_TO_FILE, - save_file_name: str = SAVE_FILE_NAME, - upload_video: bool = UPLOAD_TO_YOUTUBE, - delete_clips: bool = DELETE_CLIPS, - # youtube - title: str = TITLE, - description: str = DESCRIPTION, - thumbnail: str = THUMBNAIL, - tags: list = TAGS, + # required + data: list = DATA, + blacklist: list = BLACKLIST, + # other + path: str = get_path(), + check_version: bool = CHECK_VERSION, + # twitch + client_id: str = CLIENT_ID, + oauth_token: str = OAUTH_TOKEN, + period: int = PERIOD, + language: str = LANGUAGE, + limit: int = LIMIT, + # selenium + profile_path: str = ROOT_PROFILE_PATH, + executable_path: str = EXECUTABLE_PATH, + sleep: int = SLEEP, + headless: bool = HEADLESS, + debug: bool = DEBUG, + # video options + render_video: bool = RENDER_VIDEO, + file_name: str = FILE_NAME, + resolution: tuple = RESOLUTION, + frames: int = FRAMES, + video_length: float = VIDEO_LENGTH, + resize_clips: bool = RESIZE_CLIPS, + enable_intro: bool = ENABLE_INTRO, + resize_intro: bool = RESIZE_INTRO, + intro_path: str = INTRO_FILE_PATH, + enable_transition: bool = ENABLE_TRANSITION, + resize_transition: bool = RESIZE_TRANSITION, + transition_path: str = TRANSITION_FILE_PATH, + enable_outro: bool = ENABLE_OUTRO, + resize_outro: bool = RESIZE_OUTRO, + outro_path: str = OUTRO_FILE_PATH, + # other options + save_file: bool = SAVE_TO_FILE, + save_file_name: str = SAVE_FILE_NAME, + upload_video: bool = UPLOAD_TO_YOUTUBE, + delete_clips: bool = DELETE_CLIPS, + # youtube + title: str = TITLE, + description: str = DESCRIPTION, + thumbnail: str = THUMBNAIL, + tags: list = TAGS, ) -> None: if check_version: try: for project, version in zip( - [ - "twitchtube", - "opplast", - ], - [ - twitchtube_version, - opplast_version, - ], + [ + "twitchtube", + "opplast", + ], + [ + twitchtube_version, + opplast_version, + ], ): current = get_current_version(project) @@ -109,12 +109,10 @@ def make_video( if did_remove: log.info("Data included blacklisted content and was removed") - data = convert_name_to_ids(data, oauth_token=oauth_token, client_id=client_id) + data = name_to_ids(data, oauth_token=oauth_token, client_id=client_id) # first we get all the clips for every entry in data - for entry in data: - category, id_, name = entry - + for category, id_, name in data: # so we dont add the same clip twice new_clips, new_ids, new_titles = get_clips( blacklist, @@ -238,20 +236,20 @@ def add_clip(path: str, resolution: tuple, resize: bool = True) -> VideoFileClip def render( - path: str, - file_name: str, - resolution: tuple, - frames: int, - resize_clips: bool, - enable_intro: bool, - resize_intro: bool, - intro_path: str, - enable_transition: bool, - resize_transition: bool, - transition_path: str, - enable_outro: bool, - resize_outro: bool, - outro_path: str, + path: str, + file_name: str, + resolution: tuple, + frames: int, + resize_clips: bool, + enable_intro: bool, + resize_intro: bool, + intro_path: str, + enable_transition: bool, + resize_transition: bool, + transition_path: str, + enable_outro: bool, + resize_outro: bool, + outro_path: str, ) -> None: """ Concatenates a video with given path.