From 89f0bd7b50aefed5711ffbe90293f0ba4a1c3409 Mon Sep 17 00:00:00 2001 From: Younes Aassila <47226184+younesaassila@users.noreply.github.com> Date: Fri, 22 Dec 2023 18:34:18 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20WIP=20Most=20of=20the=20logic=20?= =?UTF-8?q?is=20there,=20except=20triggering=20replacement=20on=20ad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/page/getFetch.ts | 307 +++++++++++++++++++++++++++++++++++++------ src/page/page.ts | 17 +++ src/page/worker.ts | 24 ++++ 3 files changed, 311 insertions(+), 37 deletions(-) diff --git a/src/page/getFetch.ts b/src/page/getFetch.ts index e41814de..9c7fc35e 100644 --- a/src/page/getFetch.ts +++ b/src/page/getFetch.ts @@ -1,4 +1,5 @@ import acceptFlag from "../common/ts/acceptFlag"; +import findChannelFromTwitchTvUrl from "../common/ts/findChannelFromTwitchTvUrl"; import findChannelFromUsherUrl from "../common/ts/findChannelFromUsherUrl"; import generateRandomString from "../common/ts/generateRandomString"; import getHostFromUrl from "../common/ts/getHostFromUrl"; @@ -17,6 +18,20 @@ export interface FetchOptions { scope: "page" | "worker"; shouldWaitForStore: boolean; state?: State; + sendMessageToWorkers?: (message: any) => void; +} +export interface PlaybackAccessToken { + value: string; + signature: string; + authorization: { + isForbidden: boolean; + forbiddenReasonCode: string; + }; + __typename: string; +} +enum MessageType { + NewPlaybackAccessToken = "TLP_NewPlaybackAccessToken", + NewPlaybackAccessTokenResponse = "TLP_NewPlaybackAccessTokenResponse", } export function getFetch(options: FetchOptions): typeof fetch { @@ -25,12 +40,62 @@ export function getFetch(options: FetchOptions): typeof fetch { const videoWeaverUrlsToFlag = new Map(); // Video Weaver URLs to flag -> number of times flagged. const videoWeaverUrlsToIgnore = new Set(); // No response check. + let cachedPlaybackTokenRequestHeaders: Map | null = null; + let cachedPlaybackTokenRequestBody: string | null = null; + let cachedUsherRequestUrl: string | null = null; + let currentVideoWeaverUrls: string[] | null = null; + let replacementVideoWeaverUrls: string[] | null = null; + if (options.shouldWaitForStore) { setTimeout(() => { options.shouldWaitForStore = false; }, 5000); } + if (options.scope === "page") { + self.addEventListener("message", async event => { + if (event.data?.type === MessageType.NewPlaybackAccessToken) { + const newPlaybackAccessToken = + await fetchReplacementPlaybackAccessToken( + cachedPlaybackTokenRequestHeaders, + cachedPlaybackTokenRequestBody + ); + const message = { + type: MessageType.NewPlaybackAccessTokenResponse, + newPlaybackAccessToken, + }; + console.log("[TTV LOL PRO] πŸ’¬ Sent message to workers", message); + options.sendMessageToWorkers?.(message); + } + }); + } + + // TEST CODE + if (options.scope === "worker") { + setTimeout(async () => { + try { + console.log("[TTV LOL PRO] πŸ”„ Checking for new Video Weaver URLs…"); + const newVideoWeaverUrls = await fetchReplacementVideoWeaverUrls( + cachedUsherRequestUrl + ); + if (newVideoWeaverUrls == null) { + console.log("[TTV LOL PRO] πŸ”„ No new Video Weaver URLs found."); + return; + } + replacementVideoWeaverUrls = newVideoWeaverUrls; + console.log( + "[TTV LOL PRO] πŸ”„ Found new Video Weaver URLs:", + replacementVideoWeaverUrls + ); + } catch (error) { + console.error( + "[TTV LOL PRO] πŸ”„ Failed to get new Video Weaver URLs:", + error + ); + } + }, 30000); + } + return async function fetch( input: RequestInfo | URL, init?: RequestInit @@ -118,6 +183,8 @@ export function getFetch(options: FetchOptions): typeof fetch { } } flagRequest(headersMap); + // cachedPlaybackTokenRequestHeaders = headersMap; + // cachedPlaybackTokenRequestBody = requestBody; } else if ( requestBody != null && requestBody.includes("PlaybackAccessToken") @@ -126,19 +193,46 @@ export function getFetch(options: FetchOptions): typeof fetch { "[TTV LOL PRO] πŸ₯… Caught GraphQL PlaybackAccessToken request. Flagging…" ); flagRequest(headersMap); + // cachedPlaybackTokenRequestHeaders = headersMap; + // cachedPlaybackTokenRequestBody = requestBody; } } // Usher requests. if (host != null && usherHostRegex.test(host)) { console.debug("[TTV LOL PRO] πŸ₯… Caught Usher request."); + cachedUsherRequestUrl = url; } + let response: Response; + // Video Weaver requests. if (host != null && videoWeaverHostRegex.test(host)) { - const isIgnoredUrl = videoWeaverUrlsToIgnore.has(url); - const isNewUrl = !knownVideoWeaverUrls.has(url); - const isFlaggedUrl = videoWeaverUrlsToFlag.has(url); + let videoWeaverUrl = url; + if (replacementVideoWeaverUrls != null) { + // const index = currentVideoWeaverUrls?.findIndex( + // url => + // url.split("playlist/")[1] === videoWeaverUrl.split("playlist/")[1] + // ); + // console.log("index = ", index); + // if (index != null && index >= 0) { + // videoWeaverUrl = + // replacementVideoWeaverUrls[ + // Math.min(index, replacementVideoWeaverUrls.length - 1) + // ]; + // console.log( + // `[TTV LOL PRO] πŸ”„ Replaced Video Weaver URL '${url}' with '${videoWeaverUrl}'.` + // ); + // } + videoWeaverUrl = replacementVideoWeaverUrls[0]; + console.log( + `[TTV LOL PRO] πŸ”„ Replaced Video Weaver URL '${url}' with '${videoWeaverUrl}'.` + ); + } + + const isIgnoredUrl = videoWeaverUrlsToIgnore.has(videoWeaverUrl); + const isNewUrl = !knownVideoWeaverUrls.has(videoWeaverUrl); + const isFlaggedUrl = videoWeaverUrlsToFlag.has(videoWeaverUrl); if (!isIgnoredUrl && (isNewUrl || isFlaggedUrl)) { console.log( @@ -149,21 +243,26 @@ export function getFetch(options: FetchOptions): typeof fetch { }. Flagging…` ); flagRequest(headersMap); - videoWeaverUrlsToFlag.set( - url, - (videoWeaverUrlsToFlag.get(url) ?? 0) + 1 - ); - if (isNewUrl) knownVideoWeaverUrls.add(url); + // videoWeaverUrlsToFlag.set( + // videoWeaverUrl, + // (videoWeaverUrlsToFlag.get(videoWeaverUrl) ?? 0) + 1 + // ); + if (isNewUrl) knownVideoWeaverUrls.add(videoWeaverUrl); } + + response = await NATIVE_FETCH(videoWeaverUrl, { + ...init, + headers: Object.fromEntries(headersMap), + }); + } else { + response = await NATIVE_FETCH(input, { + ...init, + headers: Object.fromEntries(headersMap), + }); } //#endregion - const response = await NATIVE_FETCH(input, { - ...init, - headers: Object.fromEntries(headersMap), - }); - // Reading the response body can be expensive, so we only do it if we need to. let responseBody: string | undefined = undefined; const readResponseBody = async (): Promise => { @@ -196,36 +295,40 @@ export function getFetch(options: FetchOptions): typeof fetch { // Video Weaver responses. if (host != null && videoWeaverHostRegex.test(host)) { responseBody = await readResponseBody(); + currentVideoWeaverUrls = responseBody + .split("\n") + .filter(line => videoWeaverUrlRegex.test(line)); + replacementVideoWeaverUrls = null; // Check if response contains ad. if (responseBody.includes("stitched-ad")) { console.log( "[TTV LOL PRO] πŸ₯… Caught Video Weaver response containing ad." ); - if (videoWeaverUrlsToIgnore.has(url)) return response; - if (!videoWeaverUrlsToFlag.has(url)) { - // Let's proxy the next request for this URL, 2 attempts left. - videoWeaverUrlsToFlag.set(url, 0); - cancelRequest(); - } - // FIXME: This workaround doesn't work. Let's find another way. - // 0: First attempt, not proxied, cancelled. - // 1: Second attempt, proxied, cancelled. - // 2: Third attempt, proxied, last attempt by Twitch client. - // If the third attempt contains an ad, we have to let it through. - const isCancellable = videoWeaverUrlsToFlag.get(url)! < 2; - if (isCancellable) { - cancelRequest(); - } else { - console.error( - "[TTV LOL PRO] ❌ Could not cancel Video Weaver response containing ad. All attempts used." - ); - videoWeaverUrlsToFlag.delete(url); // Clear attempts. - videoWeaverUrlsToIgnore.add(url); // Ignore this URL, there's nothing we can do. - } + // if (videoWeaverUrlsToIgnore.has(url)) return response; + // if (!videoWeaverUrlsToFlag.has(url)) { + // // Let's proxy the next request for this URL, 2 attempts left. + // videoWeaverUrlsToFlag.set(url, 0); + // cancelRequest(); + // } + // // FIXME: This workaround doesn't work. Let's find another way. + // // 0: First attempt, not proxied, cancelled. + // // 1: Second attempt, proxied, cancelled. + // // 2: Third attempt, proxied, last attempt by Twitch client. + // // If the third attempt contains an ad, we have to let it through. + // const isCancellable = videoWeaverUrlsToFlag.get(url)! < 2; + // if (isCancellable) { + // cancelRequest(); + // } else { + // console.error( + // "[TTV LOL PRO] ❌ Could not cancel Video Weaver response containing ad. All attempts used." + // ); + // videoWeaverUrlsToFlag.delete(url); // Clear attempts. + // videoWeaverUrlsToIgnore.add(url); // Ignore this URL, there's nothing we can do. + // } } else { - // No ad, remove from flagged list. - videoWeaverUrlsToFlag.delete(url); - videoWeaverUrlsToIgnore.delete(url); + // // No ad, remove from flagged list. + // videoWeaverUrlsToFlag.delete(url); + // videoWeaverUrlsToIgnore.delete(url); } } @@ -345,3 +448,133 @@ function cancelRequest(): never { async function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } + +async function sendMessageToPageScript( + message: any, + timeout = 5000 +): Promise { + return new Promise((resolve, reject) => { + const listener = (event: MessageEvent) => { + if (event.data?.type === MessageType.NewPlaybackAccessTokenResponse) { + resolve(event.data?.newPlaybackAccessToken); + } + }; + self.addEventListener("message", listener); + console.log("[TTV LOL PRO] πŸ’¬ Sent message to page", message); + self.postMessage({ + type: "PageScriptMessage", + message, + }); + setTimeout(() => { + self.removeEventListener("message", listener); + reject(new Error("Timed out.")); + }, timeout); + }); +} + +async function fetchReplacementPlaybackAccessToken( + cachedPlaybackTokenRequestHeaders: Map | null, + cachedPlaybackTokenRequestBody: string | null +): Promise { + let request: Request; + if ( + cachedPlaybackTokenRequestHeaders != null && + cachedPlaybackTokenRequestBody != null + ) { + request = new Request("https://gql.twitch.tv/gql", { + method: "POST", + headers: Object.fromEntries(cachedPlaybackTokenRequestHeaders), // Headers already flagged. + body: cachedPlaybackTokenRequestBody, + }); + } else { + const login = findChannelFromTwitchTvUrl(location.href); + if (login == null) return null; + request = new Request("https://gql.twitch.tv/gql", { + method: "POST", + headers: { + // TODO: Find unnecessary headers. + Accept: "*/*", + "Accept-Language": "en-US", + "Accept-Encoding": "gzip, deflate, br", + Referer: "https://www.twitch.tv/", + Authorization: "undefined", + "Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko", + "Content-Type": "text/plain; charset=UTF-8", + "Device-ID": "umQiGH8XN9QN2A9VyHQJv7437IbqHZLL", + Origin: "https://www.twitch.tv", + DNT: "1", + Connection: "keep-alive", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + }, + body: JSON.stringify({ + operationName: "PlaybackAccessToken_Template", + query: + 'query PlaybackAccessToken_Template($login: String!, $isLive: Boolean!, $vodID: ID!, $isVod: Boolean!, $playerType: String!) { streamPlaybackAccessToken(channelName: $login, params: {platform: "web", playerBackend: "mediaplayer", playerType: $playerType}) @include(if: $isLive) { value signature authorization { isForbidden forbiddenReasonCode } __typename } videoPlaybackAccessToken(id: $vodID, params: {platform: "web", playerBackend: "mediaplayer", playerType: $playerType}) @include(if: $isVod) { value signature __typename }}', + // TODO: Check impact on VODs. + variables: { + isLive: true, + login, + isVod: false, + vodID: "", + playerType: "site", + }, + }), + }); + } + const response = await NATIVE_FETCH(request); + const json = await response.json(); + const newPlaybackAccessToken = json?.data?.streamPlaybackAccessToken; + if (newPlaybackAccessToken == null) return null; + return newPlaybackAccessToken; +} + +function getReplacementUsherUrl( + cachedUsherRequestUrl: string | null, + playbackAccessToken: PlaybackAccessToken | null +): string | null { + if (cachedUsherRequestUrl == null) return null; + if (playbackAccessToken == null) return null; + try { + const newUsherUrl = new URL(cachedUsherRequestUrl); + newUsherUrl.searchParams.delete("acmb"); + newUsherUrl.searchParams.set("play_session_id", generateRandomString(32)); + newUsherUrl.searchParams.set("sig", playbackAccessToken.signature); + newUsherUrl.searchParams.set("token", playbackAccessToken.value); + return newUsherUrl.toString(); + } catch { + return null; + } +} + +async function fetchReplacementVideoWeaverUrls( + cachedUsherRequestUrl: string | null +): Promise { + if (cachedUsherRequestUrl == null) return null; + try { + const newPlaybackAccessToken = await sendMessageToPageScript({ + type: MessageType.NewPlaybackAccessToken, + }); + if (newPlaybackAccessToken == null) { + console.log("[TTV LOL PRO] πŸ”„ No new playback token found."); + return null; + } + const newUsherUrl = getReplacementUsherUrl( + cachedUsherRequestUrl, + newPlaybackAccessToken + ); + if (newUsherUrl == null) { + console.log("[TTV LOL PRO] πŸ”„ No new Usher URL found."); + return null; + } + const response = await NATIVE_FETCH(newUsherUrl); + const text = await response.text(); + const videoWeaverUrls = text + .split("\n") + .filter(line => videoWeaverUrlRegex.test(line)); + return videoWeaverUrls; + } catch { + return null; + } +} diff --git a/src/page/page.ts b/src/page/page.ts index 6e005f42..ce012227 100644 --- a/src/page/page.ts +++ b/src/page/page.ts @@ -6,8 +6,11 @@ const params = JSON.parse(document.currentScript!.dataset.params!); const options: FetchOptions = { scope: "page", shouldWaitForStore: params.isChromium === false, + sendMessageToWorkers, }; +let workers = [] as Worker[]; + window.fetch = getFetch(options); window.Worker = class Worker extends window.Worker { @@ -52,9 +55,19 @@ window.Worker = class Worker extends window.Worker { window.postMessage(event.data.message); } }); + workers.push(this); } }; +function sendMessageToWorkers(message: any) { + workers.forEach(worker => worker.postMessage(message)); +} + +// // Ping workers every 5 seconds. +// setInterval(() => { +// sendMessageToWorkers({ type: "TLP_Ping" }); +// }, 5000); + window.addEventListener("message", event => { if (event.data?.type === "PageScriptMessage") { const message = event.data.message; @@ -66,6 +79,10 @@ window.addEventListener("message", event => { options.state = message.state; options.shouldWaitForStore = false; } + } else if (event.data?.type === "TLP_Ping") { + sendMessageToWorkers({ type: "TLP_Pong" }); + } else if (event.data?.type === "TLP_Pong") { + console.log("[TTV LOL PRO] πŸ“ Worker responded to ping."); } }); diff --git a/src/page/worker.ts b/src/page/worker.ts index 5ea2287b..79261b21 100644 --- a/src/page/worker.ts +++ b/src/page/worker.ts @@ -7,4 +7,28 @@ const options: FetchOptions = { shouldWaitForStore: false, }; +self.addEventListener("message", event => { + // console.log("[TTV LOL PRO] RECEIVED MESSAGE FROM PAGE", event.data); + if (event.data?.type === "TLP_Ping") { + self.postMessage({ + type: "PageScriptMessage", + message: { + type: "TLP_Pong", + }, + }); + } else if (event.data?.type === "TLP_Pong") { + console.log("[TTV LOL PRO] πŸ“ Page responded to ping."); + } +}); + +// // Ping page script every 5 seconds. +// setInterval(() => { +// self.postMessage({ +// type: "PageScriptMessage", +// message: { +// type: "TLP_Ping", +// }, +// }); +// }, 5000); + self.fetch = getFetch(options);