Merge pull request #165 from blackjack4494/master

Release 2020.11.11 Happy Singles Day

.github/workflows/build.yml

@@ -82,7 +82,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-          python-version: '3.x'
+          python-version: '3.8'
     - name: Install Requirements
       run: pip install pyinstaller
     - name: Bump version
@@ -116,7 +116,7 @@ jobs:
     - name: Set up Python 3.5.4 32-Bit
       uses: actions/setup-python@v2
       with:
-          python-version: '3.5.4'
+          python-version: '3.4.4'
           architecture: 'x86'
     - name: Install Requirements for 32 Bit
       run: pip install pyinstaller==3.5

youtube_dlc/downloader/common.py

@@ -364,8 +364,10 @@ def download(self, filename, info_dict, subtitle=False):
                         else '%.2f' % sleep_interval))
                 time.sleep(sleep_interval)
         else:
-            if self.params.get('sleep_interval_subtitles') > 0:
+            sleep_interval_sub = 0
+            if type(self.params.get('sleep_interval_subtitles')) is int:
                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
+            if sleep_interval_sub > 0:
                 self.to_screen(
                     '[download] Sleeping %s seconds...' % (
                         sleep_interval_sub))

youtube_dlc/downloader/external.py

@@ -115,8 +115,10 @@ class CurlFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename]
-        for key, val in info_dict['http_headers'].items():
-            cmd += ['--header', '%s: %s' % (key, val)]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
+
         cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
         cmd += self._valueless_option('--silent', 'noprogress')
         cmd += self._valueless_option('--verbose', 'verbose')
@@ -150,8 +152,9 @@ class AxelFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
-        for key, val in info_dict['http_headers'].items():
-            cmd += ['-H', '%s: %s' % (key, val)]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['-H', '%s: %s' % (key, val)]
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -162,8 +165,9 @@ class WgetFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
-        for key, val in info_dict['http_headers'].items():
-            cmd += ['--header', '%s: %s' % (key, val)]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--limit-rate', 'ratelimit')
         retry = self._option('--tries', 'retries')
         if len(retry) == 2:
@@ -189,8 +193,9 @@ def _make_cmd(self, tmpfilename, info_dict):
         if dn:
             cmd += ['--dir', dn]
         cmd += ['--out', os.path.basename(tmpfilename)]
-        for key, val in info_dict['http_headers'].items():
-            cmd += ['--header', '%s: %s' % (key, val)]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
         cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
@@ -206,8 +211,10 @@ def available(cls):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
-        for key, val in info_dict['http_headers'].items():
-            cmd += ['%s:%s' % (key, val)]
+
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['%s:%s' % (key, val)]
         return cmd
 
 
@@ -253,7 +260,7 @@ def _call_downloader(self, tmpfilename, info_dict):
         # if end_time:
         #     args += ['-t', compat_str(end_time - start_time)]
 
-        if info_dict['http_headers'] and re.match(r'^https?://', url):
+        if info_dict.get('http_headers') is not None and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             headers = handle_youtubedl_headers(info_dict['http_headers'])

youtube_dlc/downloader/youtube_live_chat.py

@@ -82,7 +82,10 @@ def parse_yt_initial_data(data):
                         offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
                     processed_fragment.extend(
                         json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
-                continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation']
+                try:
+                    continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation']
+                except KeyError:
+                    continuation_id = None
 
             self._append_fragment(ctx, processed_fragment)
 

youtube_dlc/extractor/extractors.py

@@ -1175,6 +1175,7 @@
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
+from .thisvid import ThisVidIE
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import TikTokIE
 from .tinypic import TinyPicIE

youtube_dlc/extractor/thisvid.py

@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+
+
+class ThisVidIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+/?)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/videos/french-boy-pantsed/',
+        'md5': '3397979512c682f6b85b3b04989df224',
+        'info_dict': {
+            'id': '2400174',
+            'ext': 'mp4',
+            'title': 'French Boy Pantsed',
+            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://thisvid.com/embed/2400174/',
+        'md5': '3397979512c682f6b85b3b04989df224',
+        'info_dict': {
+            'id': '2400174',
+            'ext': 'mp4',
+            'title': 'French Boy Pantsed',
+            'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
+            'age_limit': 18,
+        }
+    }]
+
+    def _real_extract(self, url):
+        main_id = self._match_id(url)
+        webpage = self._download_webpage(url, main_id)
+
+        # URL decryptor was reversed from version 4.0.4, later verified working with 5.2.0 and may change in the future.
+        kvs_version = self._html_search_regex(r'<script [^>]+?src="https://thisvid\.com/player/kt_player\.js\?v=(\d+(\.\d+)+)">', webpage, 'kvs_version', fatal=False)
+        if not kvs_version.startswith("5."):
+            self.report_warning("Major version change (" + kvs_version + ") in player engine--Download may fail.")
+
+        title = self._html_search_regex(r'<title>(?:Video: )?(.+?)(?: - (?:\w+ porn at )?ThisVid(?:.com| tube))?</title>', webpage, 'title')
+        # video_id, video_url and license_code from the 'flashvars' JSON object:
+        video_id = self._html_search_regex(r"video_id: '([0-9]+)',", webpage, 'video_id')
+        video_url = self._html_search_regex(r"video_url: '(function/0/.+?)',", webpage, 'video_url')
+        license_code = self._html_search_regex(r"license_code: '([0-9$]{16})',", webpage, 'license_code')
+        thumbnail = self._html_search_regex(r"preview_url: '((?:https?:)?//media.thisvid.com/.+?.jpg)',", webpage, 'thumbnail', fatal=False)
+        if thumbnail.startswith("//"):
+            thumbnail = "https:" + thumbnail
+        if (re.match(self._VALID_URL, url).group('type') == "videos"):
+            display_id = main_id
+        else:
+            display_id = self._search_regex(r'<link rel="canonical" href="' + self._VALID_URL + r'">', webpage, 'display_id', fatal=False),
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'url': getrealurl(video_url, license_code),
+            'thumbnail': thumbnail,
+            'age_limit': 18,
+        }
+
+
+def getrealurl(video_url, license_code):
+    urlparts = video_url.split('/')[2:]
+    license = getlicensetoken(license_code)
+    newmagic = urlparts[5][:32]
+
+    for o in range(len(newmagic) - 1, -1, -1):
+        new = ""
+        l = (o + sum([int(n) for n in license[o:]])) % 32
+
+        for i in range(0, len(newmagic)):
+            if i == o:
+                new += newmagic[l]
+            elif i == l:
+                new += newmagic[o]
+            else:
+                new += newmagic[i]
+        newmagic = new
+
+    urlparts[5] = newmagic + urlparts[5][32:]
+    return "/".join(urlparts)
+
+
+def getlicensetoken(license):
+    modlicense = license.replace("$", "").replace("0", "1")
+    center = int(len(modlicense) / 2)
+    fronthalf = int(modlicense[:center + 1])
+    backhalf = int(modlicense[center:])
+
+    modlicense = str(4 * abs(fronthalf - backhalf))
+    retval = ""
+    for o in range(0, center + 1):
+        for i in range(1, 5):
+            retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
+    return retval

youtube_dlc/extractor/viki.py

@@ -308,17 +308,26 @@ def _real_extract(self, url):
                 'url': thumbnail.get('url'),
             })
 
-        new_video = self._download_json(
-            'https://www.viki.com/api/videos/%s' % video_id, video_id,
-            'Downloading new video JSON to get subtitles', headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404])
-
         subtitles = {}
-        for sub in new_video.get('streamSubtitles').get('dash'):
-            subtitles[sub.get('srclang')] = [{
-                'ext': 'vtt',
-                'url': sub.get('src'),
-                'completion': sub.get('percentage'),
-            }]
+        try:
+            # New way to fetch subtitles
+            new_video = self._download_json(
+                'https://www.viki.com/api/videos/%s' % video_id, video_id,
+                'Downloading new video JSON to get subtitles', headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404])
+            for sub in new_video.get('streamSubtitles').get('dash'):
+                subtitles[sub.get('srclang')] = [{
+                    'ext': 'vtt',
+                    'url': sub.get('src'),
+                    'completion': sub.get('percentage'),
+                }]
+        except AttributeError:
+            # fall-back to the old way if there isn't a streamSubtitles attribute
+            for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
+                subtitles[subtitle_lang] = [{
+                    'ext': subtitles_format,
+                    'url': self._prepare_call(
+                        'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
+                } for subtitles_format in ('srt', 'vtt')]
 
         result = {
             'id': video_id,