From 49037ee26a7f87de9e86750fb20e7ddbe89950d8 Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Sun, 18 Oct 2020 17:30:50 +0200 Subject: [PATCH 1/9] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8bee628..a9825c7 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,7 @@ dmypy.json .pytype/ # Added +yotter-config.yaml data_export.json .env .envo From bcf4def8a4ce15565c45be488c793b67adc33034 Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Sun, 18 Oct 2020 18:06:12 +0200 Subject: [PATCH 2/9] config json->yaml --- app/routes.py | 3 ++- requirements.txt | 1 + yotter-config.json | 15 --------------- yotter-config.sample.yaml | 15 +++++++++++++++ 4 files changed, 18 insertions(+), 16 deletions(-) delete mode 100644 yotter-config.json create mode 100644 yotter-config.sample.yaml diff --git a/app/routes.py b/app/routes.py index 6173108..dd9c59a 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,5 +1,6 @@ import datetime import glob +import yaml import json import math import os @@ -39,7 +40,7 @@ ########################## #### Config variables #### ########################## -config = json.load(open('yotter-config.json')) +config = yaml.safe_load(open('yotter-config.yaml')) ########################## #### Config variables #### ########################## diff --git a/requirements.txt b/requirements.txt index 33bbf16..36e1326 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ pylint==2.6.0 PyMySQL==0.10.1 pyparsing==2.4.7 PySocks==1.7.1 +pyyaml==5.3.1 python-dateutil==2.8.1 python-dotenv==0.14.0 python-editor==1.0.4 diff --git a/yotter-config.json b/yotter-config.json deleted file mode 100644 index e0ceeab..0000000 --- a/yotter-config.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "serverName": "yotter.xyz", - "nitterInstance": "https://nitter.net/", - "maxInstanceUsers": 100, - "serverLocation": "Germany", - "restrictPublicUsage":true, - "nginxVideoStream":true, - "maintenance_mode":false, - "show_admin_message":false, - "admin_message_title":"Message from the admin", - "admin_message":"Message from the admin text", - "admin_user":"admin_username", - "max_old_user_days": 60, - "donate_url": "" -} diff --git a/yotter-config.sample.yaml b/yotter-config.sample.yaml new file mode 100644 index 0000000..f84a327 --- /dev/null +++ b/yotter-config.sample.yaml @@ -0,0 +1,15 @@ + +serverName: "yotter.xyz" +nitterInstance: "https://nitter.net/" +maxInstanceUsers: 100 +serverLocation: "Germany" +restrictPublicUsage: true +nginxVideoStream: true +maintenance_mode: false +show_admin_message: false +admin_message_title: "Message from the admin" +admin_message: "Message from the admin text" +admin_user: "admin_username" +max_old_user_days: 60 +donate_url: "" + From 9b71bd67913691e127275ddecb2655abfdb226cf Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 14:13:54 +0200 Subject: [PATCH 3/9] remove leading `/` from urls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the `youtube` module shouldn’t concern itself with proxying/routing details --- youtube/channel.py | 10 ---------- youtube/channels.py | 2 +- youtube/playlist.py | 14 ++++++-------- youtube/search.py | 30 ++++++++++++------------------ youtube/util.py | 24 ++---------------------- youtube/watch.py | 2 +- 6 files changed, 22 insertions(+), 60 deletions(-) diff --git a/youtube/channel.py b/youtube/channel.py index b1d991d..a87eb6f 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -180,21 +180,12 @@ def get_channel_search_json(channel_id, query, page): ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii') polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, headers_desktop, debug_name='channel_search') - return polymer_json - def post_process_channel_info(info): - info['avatar'] = util.prefix_url(info['avatar']) - info['channel_url'] = util.prefix_url(info['channel_url']) for item in info['items']: - util.prefix_urls(item) util.add_extra_html_info(item) - - - - playlist_sort_codes = {'2': "da", '3': "dd", '4': "lad"} # youtube.com/[channel_id]/[tab] @@ -202,7 +193,6 @@ def post_process_channel_info(info): # youtube.com/c/[custom]/[tab] # youtube.com/[custom]/[tab] def get_channel_page_general_url(base_url, tab, request, channel_id=None): - page_number = int(request.args.get('page', 1)) sort = request.args.get('sort', '3') view = request.args.get('view', '1') diff --git a/youtube/channels.py b/youtube/channels.py index 5c45ef7..98261f7 100644 --- a/youtube/channels.py +++ b/youtube/channels.py @@ -172,7 +172,7 @@ def get_author_info_from_channel(content): channel = { "channelId": cmd['channelId'], "username": cmd['title'], - "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url'].replace("/", "~")), + "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url']), "description":description, "suscribers": cmd['subscriberCountText']['runs'][0]['text'].split(" ")[0], "banner": cmd['banner']['thumbnails'][0]['url'] diff --git a/youtube/playlist.py b/youtube/playlist.py index 2e2af98..8225e82 100644 --- a/youtube/playlist.py +++ b/youtube/playlist.py @@ -14,15 +14,15 @@ -def playlist_ctoken(playlist_id, offset): - +def playlist_ctoken(playlist_id, offset): + offset = proto.uint(1, offset) # this is just obfuscation as far as I can tell. It doesn't even follow protobuf offset = b'PT:' + proto.unpadded_b64encode(offset) offset = proto.string(15, offset) continuation_info = proto.string( 3, proto.percent_b64encode(offset) ) - + playlist_id = proto.string(2, 'VL' + playlist_id ) pointless_nest = proto.string(80226972, playlist_id + continuation_info) @@ -51,7 +51,7 @@ def playlist_first_page(playlist_id, report_text = "Retrieved playlist"): content = json.loads(util.uppercase_escape(content.decode('utf-8'))) return content - + #https://m.youtube.com/playlist?itct=CBMQybcCIhMIptj9xJaJ2wIV2JKcCh3Idwu-&ctoken=4qmFsgI2EiRWTFBMT3kwajlBdmxWWlB0bzZJa2pLZnB1MFNjeC0tN1BHVEMaDmVnWlFWRHBEUWxFJTNE&pbj=1 def get_videos(playlist_id, page): @@ -84,7 +84,7 @@ def get_playlist_page(): this_page_json = first_page_json else: tasks = ( - gevent.spawn(playlist_first_page, playlist_id, report_text="Retrieved playlist info" ), + gevent.spawn(playlist_first_page, playlist_id, report_text="Retrieved playlist info" ), gevent.spawn(get_videos, playlist_id, page) ) gevent.joinall(tasks) @@ -98,12 +98,10 @@ def get_playlist_page(): if page != '1': info['metadata'] = yt_data_extract.extract_playlist_metadata(first_page_json) - util.prefix_urls(info['metadata']) for item in info.get('items', ()): - util.prefix_urls(item) util.add_extra_html_info(item) if 'id' in item: - item['thumbnail'] = '/https://i.ytimg.com/vi/' + item['id'] + '/default.jpg' + item['thumbnail'] = 'https://i.ytimg.com/vi/' + item['id'] + '/default.jpg' item['url'] += '&list=' + playlist_id if item['index']: diff --git a/youtube/search.py b/youtube/search.py index 7888104..ebbc4a1 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -61,28 +61,22 @@ def get_channel_renderer_item_info(item): suscribers = item['subscriberCountText']['simpleText'].split(" ")[0] except: suscribers = "?" - + try: description = utils.get_description_snippet_text(item['descriptionSnippet']['runs']) except KeyError: description = "" + channel = { + "channelId": item['channelId'], + "username": item['title']['simpleText'], + "description": Markup(str(description)), + "suscribers": suscribers, + "thumbnail": "https:{}".format(item['thumbnail']['thumbnails'][0]['url']), + } + + try: channel["videos"] = item['videoCountText']['runs'][0]['text'] + except KeyError: pass - try: - channel = { - "channelId": item['channelId'], - "username": item['title']['simpleText'], - "thumbnail": "https:{}".format(item['thumbnail']['thumbnails'][0]['url'].replace("/", "~")), - "description": Markup(str(description)), - "suscribers": suscribers, - "videos": item['videoCountText']['runs'][0]['text'] - } - except KeyError: - channel = { - "channelId": item['channelId'], - "username": item['title']['simpleText'], - "avatar": item['thumbnail']['thumbnails'][0]['url'], - "suscribers": suscribers - } return channel def get_videos_from_search(search): @@ -105,7 +99,7 @@ def get_videos_from_search(search): except KeyError: continue - # Sometimes Youtube will return an empty query. Try again. + # Sometimes Youtube will return an empty query. Try again. return results def get_video_renderer_item_info(item): diff --git a/youtube/util.py b/youtube/util.py index e3f6c65..3e59f20 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -49,7 +49,7 @@ import urllib3 import urllib3.contrib.socks -URL_ORIGIN = "/https://www.youtube.com" +URL_ORIGIN = "https://www.youtube.com" connection_pool = urllib3.PoolManager(cert_reqs = 'CERT_REQUIRED') @@ -304,7 +304,7 @@ def video_id(url): # default, sddefault, mqdefault, hqdefault, hq720 def get_thumbnail_url(video_id): return "/i.ytimg.com/vi/" + video_id + "/mqdefault.jpg" - + def seconds_to_timestamp(seconds): seconds = int(seconds) hours, seconds = divmod(seconds,3600) @@ -332,12 +332,6 @@ def uppercase_escape(s): r'\\U([0-9a-fA-F]{8})', lambda m: chr(int(m.group(1), base=16)), s) -def prefix_url(url): - if url is None: - return None - url = url.lstrip('/') # some urls have // before them, which has a special meaning - return '/' + url - def left_remove(string, substring): '''removes substring from the start of string, if present''' if string.startswith(substring): @@ -354,21 +348,9 @@ def concat_or_none(*strings): return result -def prefix_urls(item): - try: - item['thumbnail'] = prefix_url(item['thumbnail']) - except KeyError: - pass - - try: - item['author_url'] = prefix_url(item['author_url']) - except KeyError: - pass - def add_extra_html_info(item): if item['type'] == 'video': item['url'] = (URL_ORIGIN + '/watch?v=' + item['id']) if item.get('id') else None - video_info = {} for key in ('id', 'title', 'author', 'duration'): try: @@ -385,9 +367,7 @@ def add_extra_html_info(item): def parse_info_prepare_for_html(renderer, additional_info={}): item = yt_data_extract.extract_item_info(renderer, additional_info) - prefix_urls(item) add_extra_html_info(item) - return item def check_gevent_exceptions(*tasks): diff --git a/youtube/watch.py b/youtube/watch.py index d974665..dcaaf42 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -34,7 +34,7 @@ def make_caption_src(info, lang, auto=False, trans_lang=None): if trans_lang: label += ' -> ' + trans_lang return { - 'url': '/' + yt_data_extract.get_caption_url(info, lang, 'vtt', auto, trans_lang), + 'src': yt_data_extract.get_caption_url(info, lang, 'vtt', auto, trans_lang), 'label': label, 'srclang': trans_lang[0:2] if trans_lang else lang[0:2], 'on': False, From 8c7bec67a1df52d7b84d8cf505fbb6faf8ead72d Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 15:54:57 +0200 Subject: [PATCH 4/9] rework and simplify internal+external proxying --- app/routes.py | 61 +++++++++++++++++++------------------ app/templates/video.html | 10 +++--- app/templates/ytsearch.html | 12 +++----- yotter-config.sample.yaml | 21 ++++++++++++- 4 files changed, 59 insertions(+), 45 deletions(-) diff --git a/app/routes.py b/app/routes.py index dd9c59a..3b043bb 100644 --- a/app/routes.py +++ b/app/routes.py @@ -336,15 +336,11 @@ def ytsearch(): prev_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=int(page) - 1) for video in results['videos']: - hostname = urllib.parse.urlparse(video['videoThumb']).netloc - video['videoThumb'] = video['videoThumb'].replace("https://{}".format(hostname), "") + "&host=" + hostname + video['videoThumb'] = proxy_image_url(video['videoThumb']) for channel in results['channels']: - if config['nginxVideoStream']: - channel['thumbnail'] = channel['thumbnail'].replace("~", "/") - hostName = urllib.parse.urlparse(channel['thumbnail']).netloc - channel['thumbnail'] = channel['thumbnail'].replace("https://{}".format(hostName), - "") + "?host=" + hostName + channel['thumbnail'] = proxy_image_url(channel['thumbnail']) + return render_template('ytsearch.html', form=form, btform=button_form, results=results, restricted=config['restrictPublicUsage'], config=config, npage=next_page, ppage=prev_page) @@ -424,18 +420,9 @@ def channel(id): data = ytch.get_channel_tab_info(id, page, sort) for video in data['items']: - if config['nginxVideoStream']: - hostName = urllib.parse.urlparse(video['thumbnail'][1:]).netloc - video['thumbnail'] = video['thumbnail'].replace("https://{}".format(hostName), "")[1:].replace("hqdefault", - "mqdefault") + "&host=" + hostName - else: - video['thumbnail'] = video['thumbnail'].replace('/', '~') + video['thumbnail'] = proxy_image_url(video['thumbnail']) - if config['nginxVideoStream']: - hostName = urllib.parse.urlparse(data['avatar'][1:]).netloc - data['avatar'] = data['avatar'].replace("https://{}".format(hostName), "")[1:] + "?host=" + hostName - else: - data['avatar'] = data['avatar'].replace('/', '~') + data['avatar'] = proxy_image_url(data['avatar']) next_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) + 1) if int(page) == 1: @@ -483,13 +470,11 @@ def watch(): retry -= 1 for source in vsources: - hostName = urllib.parse.urlparse(source['src']).netloc - source['src'] = source['src'].replace("https://{}".format(hostName), "") + "&host=" + hostName + source['src'] = proxy_video_source_url(source['src']) # Parse video formats for v_format in info['formats']: - hostName = urllib.parse.urlparse(v_format['url']).netloc - v_format['url'] = v_format['url'].replace("https://{}".format(hostName), "") + "&host=" + hostName + v_format['url'] = proxy_video_source_url(v_format['url']) if v_format['audio_bitrate'] is not None and v_format['vcodec'] is None: v_format['audio_valid'] = True @@ -524,11 +509,10 @@ def markupString(string): ## PROXY videos through Yotter server to the client. -@app.route('/stream/', methods=['GET', 'POST']) +@app.route('/stream/', methods=['GET', 'POST']) @login_required def stream(url): # This function proxies the video stream from GoogleVideo to the client. - url = url.replace('YotterSlash', '/') headers = Headers() if (url): s = requests.Session() @@ -586,6 +570,27 @@ def img(url): pic = requests.get(url.replace("~", "/")) return Response(pic, mimetype="image/png") +# Proxy yt images through server +@app.route('/ytimg/') +@login_required +def ytimg(url): + pic = requests.get(url, stream=True) + response = Response(pic, mimetype=pic.headers['Content-Type'], direct_passthrough=True) + # extend browser file caching with etags (ytimg uses 7200) + response.cache_control.public = True + response.cache_control.max_age = int(60000) + return response + +def proxy_url(url, endpoint, config_entry): + ext_proxy = config.get('external_proxy', None) + if ext_proxy: return ext_proxy.format(**urllib.parse.urlparse(url)._asdict()) + return url_for(endpoint,url=url) if config.get(config_entry, None) else url + +def proxy_video_source_url(url): + return proxy_url(url, 'stream', 'proxy_videos') + +def proxy_image_url(url): + return proxy_url(url, 'ytimg', 'proxy_images') @app.route('/logout') def logout(): @@ -1028,12 +1033,8 @@ def getYoutubePosts(ids): video.channelUrl = vid.author_detail.href video.id = vid.yt_videoid video.videoTitle = vid.title - if config['nginxVideoStream']: - hostName = urllib.parse.urlparse(vid.media_thumbnail[0]['url']).netloc - video.videoThumb = vid.media_thumbnail[0]['url'].replace("https://{}".format(hostName), "").replace( - "hqdefault", "mqdefault") + "?host=" + hostName - else: - video.videoThumb = vid.media_thumbnail[0]['url'].replace('/', '~') + video.videoThumb = proxy_image_url(vid.media_thumbnail[0]['url']) + video.views = vid.media_statistics['views'] video.description = vid.summary_detail.value video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120] + "...", diff --git a/app/templates/video.html b/app/templates/video.html index 310f987..13804dc 100644 --- a/app/templates/video.html +++ b/app/templates/video.html @@ -40,11 +40,9 @@
Livestreams are under developent and still not supported o controls buffered preload="none"> - {% if config.nginxVideoStream %} - {% for source in vsources %} - - {% endfor %} - {% endif %} + {% for source in vsources %} + + {% endfor %} {%endif%} @@ -115,7 +113,7 @@

Comments

{% include '_video_comment.html' %} {% endfor %} - + {% if info.live %} diff --git a/app/templates/ytsearch.html b/app/templates/ytsearch.html index ccb4966..d27eca3 100644 --- a/app/templates/ytsearch.html +++ b/app/templates/ytsearch.html @@ -19,15 +19,11 @@ {% if results.channels %}

Users

{% endif %} -
+
{% for res in results.channels %}
- {% if config.nginxVideoStream %} - Avatar - {% else %} - Avatar - {% endif %} + Avatar
{{res.username}} @@ -45,7 +41,7 @@

Users

{{res.videos}}
- + {% if restricted or current_user.is_authenticated %}
{% if not current_user.is_following_yt(res.channelId) %} @@ -59,7 +55,7 @@

Users

{{ btform.submit(value='Unfollow') }} {% endif %} -
+
{% endif %}
diff --git a/yotter-config.sample.yaml b/yotter-config.sample.yaml index f84a327..260ce8a 100644 --- a/yotter-config.sample.yaml +++ b/yotter-config.sample.yaml @@ -4,7 +4,26 @@ nitterInstance: "https://nitter.net/" maxInstanceUsers: 100 serverLocation: "Germany" restrictPublicUsage: true -nginxVideoStream: true + +# Deprecated +#nginxVideoStream: true + +# Whether to proxy images (thumbnails etc.) through the server +proxy_images: true + +# Whether to proxy video (or audio) streams through the server +proxy_videos: true + +# Url for external proxy server +# If this is unset and `proxy_*` is set to true, requests will be proxied internally. +# It's recommended to use an external proxy such as nginx for better performance. +# +# Vars in {} for the original url as per `urllib.parse.urlparse`: +# scheme, netloc (host:port), path (with a leading `/`), query (without leading `?`), fragment (without leading `#`) +# +# The example below works for the nginx configuration in `SELF-HOSTING.md` +#external_proxy: "https://my.nginx.instance{path}?{query}&host={netloc}" + maintenance_mode: false show_admin_message: false admin_message_title: "Message from the admin" From 68cfd1fbe2cba67712dfed2135130f1ee12ad6be Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 16:20:23 +0200 Subject: [PATCH 5/9] add `_encoded` variants to vars for `external_proxy` config --- app/routes.py | 7 ++++++- yotter-config.sample.yaml | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/routes.py b/app/routes.py index 3b043bb..f9ea8fe 100644 --- a/app/routes.py +++ b/app/routes.py @@ -583,7 +583,12 @@ def ytimg(url): def proxy_url(url, endpoint, config_entry): ext_proxy = config.get('external_proxy', None) - if ext_proxy: return ext_proxy.format(**urllib.parse.urlparse(url)._asdict()) + if ext_proxy: + parsed = urllib.parse.urlparse(url)._asdict() + parsed['url'] = url + encoded = {key+'_encoded': urllib.parse.quote_plus(value) for (key,value) in parsed.items()} + joined = dict(parsed, **encoded) + return ext_proxy.format(**joined) return url_for(endpoint,url=url) if config.get(config_entry, None) else url def proxy_video_source_url(url): diff --git a/yotter-config.sample.yaml b/yotter-config.sample.yaml index 260ce8a..1161fae 100644 --- a/yotter-config.sample.yaml +++ b/yotter-config.sample.yaml @@ -18,10 +18,13 @@ proxy_videos: true # If this is unset and `proxy_*` is set to true, requests will be proxied internally. # It's recommended to use an external proxy such as nginx for better performance. # -# Vars in {} for the original url as per `urllib.parse.urlparse`: -# scheme, netloc (host:port), path (with a leading `/`), query (without leading `?`), fragment (without leading `#`) +# It's possible to use variables in `{}` that refer to the original request url: +# `scheme`, `netloc` (host:port), `path` (with a leading `/`), `query` (without leading `?`), `fragment` (without leading `#`): as per `urllib.parse.urlparse` +# `scheme_encoded`, `netloc_encoded` etc.: same as above but urlencoded +# `url`, `url_encoded`: the original request url, and urlencoded variant # # The example below works for the nginx configuration in `SELF-HOSTING.md` +# #external_proxy: "https://my.nginx.instance{path}?{query}&host={netloc}" maintenance_mode: false From 9344a0f5a7114cff85cf1db793e3fefa7da10b12 Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 16:39:26 +0200 Subject: [PATCH 6/9] restore `hqdefault`->`mqdefault` for thumbnails --- app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes.py b/app/routes.py index f9ea8fe..59eadc0 100644 --- a/app/routes.py +++ b/app/routes.py @@ -595,7 +595,7 @@ def proxy_video_source_url(url): return proxy_url(url, 'stream', 'proxy_videos') def proxy_image_url(url): - return proxy_url(url, 'ytimg', 'proxy_images') + return proxy_url(url, 'ytimg', 'proxy_images').replace('hqdefault','mqdefault') @app.route('/logout') def logout(): From 5f596987b236d63c27a3cd0bb02687f9604136b2 Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 20:36:10 +0200 Subject: [PATCH 7/9] reorder --- app/routes.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/app/routes.py b/app/routes.py index 59eadc0..e405b53 100644 --- a/app/routes.py +++ b/app/routes.py @@ -538,6 +538,16 @@ def download_file(streamable): for chunk in stream.iter_content(chunk_size=8192): yield chunk +# Proxy yt images through server +@app.route('/ytimg/') +@login_required +def ytimg(url): + pic = requests.get(url, stream=True) + response = Response(pic, mimetype=pic.headers['Content-Type'], direct_passthrough=True) + # extend browser file caching with etags (ytimg uses 7200) + response.cache_control.public = True + response.cache_control.max_age = int(60000) + return response ######################### #### General Logic ###### @@ -570,17 +580,6 @@ def img(url): pic = requests.get(url.replace("~", "/")) return Response(pic, mimetype="image/png") -# Proxy yt images through server -@app.route('/ytimg/') -@login_required -def ytimg(url): - pic = requests.get(url, stream=True) - response = Response(pic, mimetype=pic.headers['Content-Type'], direct_passthrough=True) - # extend browser file caching with etags (ytimg uses 7200) - response.cache_control.public = True - response.cache_control.max_age = int(60000) - return response - def proxy_url(url, endpoint, config_entry): ext_proxy = config.get('external_proxy', None) if ext_proxy: From 3914c548a9bbe2cac0be6d6227b27a235725eaef Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Mon, 19 Oct 2020 21:34:37 +0200 Subject: [PATCH 8/9] fix comment avatars --- app/routes.py | 2 ++ youtube/utils.py | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/routes.py b/app/routes.py index e405b53..c18b3b5 100644 --- a/app/routes.py +++ b/app/routes.py @@ -489,6 +489,8 @@ def watch(): videocomments = utils.post_process_comments_info(videocomments) if videocomments is not None: videocomments.sort(key=lambda x: x['likes'], reverse=True) + for cmnt in videocomments: + cmnt['thumbnail'] = proxy_image_url(cmnt['thumbnail']) # Calculate rating % if info['like_count']+info['dislike_count']>0: diff --git a/youtube/utils.py b/youtube/utils.py index 8bc0beb..1c658dc 100644 --- a/youtube/utils.py +++ b/youtube/utils.py @@ -27,11 +27,9 @@ def concat_texts(strings): def parse_comment(raw_comment): cmnt = {} - imgHostName = urllib.parse.urlparse(raw_comment['author_avatar'][1:]).netloc cmnt['author'] = raw_comment['author'] - cmnt['thumbnail'] = raw_comment['author_avatar'].replace("https://{}".format(imgHostName),"")[1:] + "?host=" + imgHostName + cmnt['thumbnail'] = raw_comment['author_avatar'] - print(cmnt['thumbnail']) cmnt['channel'] = raw_comment['author_url'] cmnt['text'] = Markup(bleach.linkify(concat_texts(raw_comment['text']).replace("\n", "
"))) cmnt['date'] = raw_comment['time_published'] From 931c3c117034ebef724c88980911467c2215e165 Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Tue, 20 Oct 2020 19:53:46 +0200 Subject: [PATCH 9/9] fix logic --- app/routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/routes.py b/app/routes.py index c18b3b5..988c0e1 100644 --- a/app/routes.py +++ b/app/routes.py @@ -583,6 +583,7 @@ def img(url): return Response(pic, mimetype="image/png") def proxy_url(url, endpoint, config_entry): + if not config.get(config_entry, None): return url ext_proxy = config.get('external_proxy', None) if ext_proxy: parsed = urllib.parse.urlparse(url)._asdict() @@ -590,7 +591,7 @@ def proxy_url(url, endpoint, config_entry): encoded = {key+'_encoded': urllib.parse.quote_plus(value) for (key,value) in parsed.items()} joined = dict(parsed, **encoded) return ext_proxy.format(**joined) - return url_for(endpoint,url=url) if config.get(config_entry, None) else url + return url_for(endpoint,url=url) def proxy_video_source_url(url): return proxy_url(url, 'stream', 'proxy_videos')