diff --git a/.github/workflows/build_pr.yml b/.github/workflows/build_pr.yml index ac657000a..47c9811f3 100644 --- a/.github/workflows/build_pr.yml +++ b/.github/workflows/build_pr.yml @@ -27,3 +27,26 @@ jobs: with: name: app path: app/build/outputs/apk/full/debug/*.apk + + build-foss: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: "zulu" + cache: 'gradle' + + - name: Build debug APK and run jvm tests + run: ./gradlew assembleFossDebug lintFossDebug testFossDebugUnitTest --stacktrace -DskipFormatKtlint + env: + PULL_REQUEST: 'true' + + - name: Upload APK + uses: actions/upload-artifact@v4 + with: + name: app-foss + path: app/build/outputs/apk/foss/debug/*.apk diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index f828764f5..a5aabb12c 100755 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -78,4 +78,11 @@ # Keep Data data classes -keep class com.my.kizzy.remote.** { ; } # Keep Gateway data classes --keep class com.my.kizzy.gateway.entities.** { ; } \ No newline at end of file +-keep class com.my.kizzy.gateway.entities.** { ; } + +## Rules for NewPipeExtractor +-keep class org.schabi.newpipe.extractor.timeago.patterns.** { *; } +-keep class org.mozilla.javascript.** { *; } +-keep class org.mozilla.classfile.ClassFileWriter +-dontwarn org.mozilla.javascript.JavaToJSONConverters +-dontwarn org.mozilla.javascript.tools.** \ No newline at end of file diff --git a/app/src/main/java/com/zionhuang/music/App.kt b/app/src/main/java/com/zionhuang/music/App.kt index fe609d1a4..f2ef3a767 100644 --- a/app/src/main/java/com/zionhuang/music/App.kt +++ b/app/src/main/java/com/zionhuang/music/App.kt @@ -94,7 +94,11 @@ class App : Application(), ImageLoaderFactory { dataStore.data .map { it[InnerTubeCookieKey] } .distinctUntilChanged() - .collect { cookie -> + .collect { rawCookie -> + // quick hack until https://github.com/z-huang/InnerTune/pull/1694 is done + val isLoggedIn: Boolean = rawCookie?.contains("SAPISID") ?: false + val cookie = if (isLoggedIn) rawCookie else null + YouTube.cookie = cookie } } diff --git a/app/src/main/java/com/zionhuang/music/playback/DownloadUtil.kt b/app/src/main/java/com/zionhuang/music/playback/DownloadUtil.kt index 42f75ab9c..136f45711 100644 --- a/app/src/main/java/com/zionhuang/music/playback/DownloadUtil.kt +++ b/app/src/main/java/com/zionhuang/music/playback/DownloadUtil.kt @@ -88,10 +88,7 @@ class DownloadUtil @Inject constructor( AudioQuality.LOW -> -1 } + (if (it.mimeType.startsWith("audio/webm")) 10240 else 0) // prefer opus stream } - }!!.let { - // Specify range to avoid YouTube's throttling - it.copy(url = "${it.url}&range=0-${it.contentLength ?: 10000000}") - } + }!! database.query { upsert( @@ -108,8 +105,13 @@ class DownloadUtil @Inject constructor( ) } - songUrlCache[mediaId] = format.url!! to playerResponse.streamingData!!.expiresInSeconds * 1000L - dataSpec.withUri(format.url!!.toUri()) + val streamUrl = format.findUrl()?.let { + // Specify range to avoid YouTube's throttling + "${it}&range=0-${format.contentLength ?: 10000000}" + } + + songUrlCache[mediaId] = streamUrl!! to playerResponse.streamingData!!.expiresInSeconds * 1000L + dataSpec.withUri(streamUrl.toUri()) } val downloadNotificationHelper = DownloadNotificationHelper(context, ExoDownloadService.CHANNEL_ID) val downloadManager: DownloadManager = DownloadManager(context, databaseProvider, downloadCache, dataSourceFactory, Executor(Runnable::run)).apply { diff --git a/app/src/main/java/com/zionhuang/music/playback/MusicService.kt b/app/src/main/java/com/zionhuang/music/playback/MusicService.kt index dbae89d14..20695bba9 100644 --- a/app/src/main/java/com/zionhuang/music/playback/MusicService.kt +++ b/app/src/main/java/com/zionhuang/music/playback/MusicService.kt @@ -691,8 +691,10 @@ class MusicService : MediaLibraryService(), } scope.launch(Dispatchers.IO) { recoverSong(mediaId, playerResponse) } - songUrlCache[mediaId] = format.url!! to playerResponse.streamingData!!.expiresInSeconds * 1000L - dataSpec.withUri(format.url!!.toUri()).subrange(dataSpec.uriPositionOffset, CHUNK_LENGTH) + val streamUrl = format.findUrl() + + songUrlCache[mediaId] = streamUrl!! to playerResponse.streamingData!!.expiresInSeconds * 1000L + dataSpec.withUri(streamUrl.toUri()).subrange(dataSpec.uriPositionOffset, CHUNK_LENGTH) } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 91c965d27..ecc303140 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -89,6 +89,10 @@ firebase-perf-plugin = { module = "com.google.firebase:perf-plugin", version = " mlkit-language-id = { group = "com.google.mlkit", name = "language-id", version = "17.0.6" } mlkit-translate = { group = "com.google.mlkit", name = "translate", version = "17.0.3" } +#newpipe-extractor = { group = "com.github.TeamNewPipe", name = "NewPipeExtractor", version = "v0.24.3" } +# Use fork of NewPipeExtractor until https://github.com/TeamNewPipe/NewPipeExtractor/pull/1253 is merged +newpipe-extractor = { group = "com.github.gechoto", name = "NewPipeExtractor", version = "0a5158d9052d57a9dbf184814de1988a8cb7824d" } + [plugins] kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } diff --git a/innertube/build.gradle.kts b/innertube/build.gradle.kts index c2828a46f..ab489df12 100644 --- a/innertube/build.gradle.kts +++ b/innertube/build.gradle.kts @@ -15,5 +15,6 @@ dependencies { implementation(libs.ktor.serialization.json) implementation(libs.ktor.client.encoding) implementation(libs.brotli) + implementation(libs.newpipe.extractor) testImplementation(libs.junit) } \ No newline at end of file diff --git a/innertube/src/main/java/com/zionhuang/innertube/InnerTube.kt b/innertube/src/main/java/com/zionhuang/innertube/InnerTube.kt index ae0775f1d..97d1c8d9a 100644 --- a/innertube/src/main/java/com/zionhuang/innertube/InnerTube.kt +++ b/innertube/src/main/java/com/zionhuang/innertube/InnerTube.kt @@ -18,6 +18,7 @@ import io.ktor.serialization.kotlinx.json.* import io.ktor.util.encodeBase64 import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.json.Json +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager import java.net.Proxy import java.util.* @@ -74,7 +75,7 @@ class InnerTube { } defaultRequest { - url("https://music.youtube.com/youtubei/v1/") + url(YouTubeClient.API_URL_YOUTUBE_MUSIC) } } @@ -82,24 +83,21 @@ class InnerTube { contentType(ContentType.Application.Json) headers { append("X-Goog-Api-Format-Version", "1") - append("X-YouTube-Client-Name", client.clientName) + append("X-YouTube-Client-Name", client.clientId) append("X-YouTube-Client-Version", client.clientVersion) - append("x-origin", "https://music.youtube.com") - if (client.referer != null) { - append("Referer", client.referer) - } - if (setLogin) { + append("X-Origin", YouTubeClient.ORIGIN_YOUTUBE_MUSIC) + append("Referer", YouTubeClient.REFERER_YOUTUBE_MUSIC) + if (setLogin && client.supportsLogin) { cookie?.let { cookie -> append("cookie", cookie) if ("SAPISID" !in cookieMap) return@let val currentTime = System.currentTimeMillis() / 1000 - val sapisidHash = sha1("$currentTime ${cookieMap["SAPISID"]} https://music.youtube.com") + val sapisidHash = sha1("$currentTime ${cookieMap["SAPISID"]} ${YouTubeClient.ORIGIN_YOUTUBE_MUSIC}") append("Authorization", "SAPISIDHASH ${currentTime}_${sapisidHash}") } } } userAgent(client.userAgent) - parameter("key", client.api_key) parameter("prettyPrint", false) } @@ -139,7 +137,13 @@ class InnerTube { } else it }, videoId = videoId, - playlistId = playlistId + playlistId = playlistId, + playbackContext = + if (client.useSignatureTimestamp) { + PlayerBody.PlaybackContext(PlayerBody.PlaybackContext.ContentPlaybackContext( + signatureTimestamp = YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId) + )) + } else null ) ) } @@ -226,7 +230,6 @@ class InnerTube { client: YouTubeClient, videoId: String, ) = httpClient.post("https://music.youtube.com/youtubei/v1/get_transcript") { - parameter("key", "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX3") headers { append("Content-Type", "application/json") } diff --git a/innertube/src/main/java/com/zionhuang/innertube/NewPipeDownloaderImpl.kt b/innertube/src/main/java/com/zionhuang/innertube/NewPipeDownloaderImpl.kt new file mode 100644 index 000000000..44f4052db --- /dev/null +++ b/innertube/src/main/java/com/zionhuang/innertube/NewPipeDownloaderImpl.kt @@ -0,0 +1,53 @@ +package com.zionhuang.innertube + +import com.zionhuang.innertube.models.YouTubeClient +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.downloader.Request +import org.schabi.newpipe.extractor.downloader.Response +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException +import java.io.IOException + +object NewPipeDownloaderImpl : Downloader() { + + private val client = OkHttpClient.Builder().build() + + @Throws(IOException::class, ReCaptchaException::class) + override fun execute(request: Request): Response { + val httpMethod = request.httpMethod() + val url = request.url() + val headers = request.headers() + val dataToSend = request.dataToSend() + + val requestBuilder = okhttp3.Request.Builder() + .method(httpMethod, dataToSend?.toRequestBody()) + .url(url) + .addHeader("User-Agent", YouTubeClient.USER_AGENT_WEB) + + headers.forEach { (headerName, headerValueList) -> + if (headerValueList.size > 1) { + requestBuilder.removeHeader(headerName) + headerValueList.forEach { headerValue -> + requestBuilder.addHeader(headerName, headerValue) + } + } else if (headerValueList.size == 1) { + requestBuilder.header(headerName, headerValueList[0]) + } + } + + val response = client.newCall(requestBuilder.build()).execute() + + if (response.code == 429) { + response.close() + + throw ReCaptchaException("reCaptcha Challenge requested", url) + } + + val responseBodyToReturn = response.body?.string() + + val latestUrl = response.request.url.toString() + return Response(response.code, response.message, response.headers.toMultimap(), responseBodyToReturn, latestUrl) + } + +} \ No newline at end of file diff --git a/innertube/src/main/java/com/zionhuang/innertube/YouTube.kt b/innertube/src/main/java/com/zionhuang/innertube/YouTube.kt index 62349e442..da0a6fed5 100644 --- a/innertube/src/main/java/com/zionhuang/innertube/YouTube.kt +++ b/innertube/src/main/java/com/zionhuang/innertube/YouTube.kt @@ -12,10 +12,10 @@ import com.zionhuang.innertube.models.SearchSuggestions import com.zionhuang.innertube.models.SongItem import com.zionhuang.innertube.models.WatchEndpoint import com.zionhuang.innertube.models.WatchEndpoint.WatchEndpointMusicSupportedConfigs.WatchEndpointMusicConfig.Companion.MUSIC_VIDEO_TYPE_ATV -import com.zionhuang.innertube.models.YouTubeClient.Companion.ANDROID_MUSIC import com.zionhuang.innertube.models.YouTubeClient.Companion.IOS import com.zionhuang.innertube.models.YouTubeClient.Companion.TVHTML5 import com.zionhuang.innertube.models.YouTubeClient.Companion.WEB +import com.zionhuang.innertube.models.YouTubeClient.Companion.WEB_CREATOR import com.zionhuang.innertube.models.YouTubeClient.Companion.WEB_REMIX import com.zionhuang.innertube.models.YouTubeLocale import com.zionhuang.innertube.models.getContinuation @@ -54,6 +54,7 @@ import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonPrimitive +import org.schabi.newpipe.extractor.NewPipe import java.net.Proxy /** @@ -63,6 +64,10 @@ import java.net.Proxy object YouTube { private val innerTube = InnerTube() + init { + NewPipe.init(NewPipeDownloaderImpl) + } + var locale: YouTubeLocale get() = innerTube.locale set(value) { @@ -431,8 +436,8 @@ object YouTube { suspend fun player(videoId: String, playlistId: String? = null): Result = runCatching { var playerResponse: PlayerResponse - if (this.cookie != null) { // if logged in: try ANDROID_MUSIC client first because IOS client does not play age restricted songs - playerResponse = innerTube.player(ANDROID_MUSIC, videoId, playlistId).body() + if (this.cookie != null) { // if logged in: try WEB_CREATOR client first because IOS client does not support login + playerResponse = innerTube.player(WEB_CREATOR, videoId, playlistId).body() if (playerResponse.playabilityStatus.status == "OK") { return@runCatching playerResponse } diff --git a/innertube/src/main/java/com/zionhuang/innertube/models/YouTubeClient.kt b/innertube/src/main/java/com/zionhuang/innertube/models/YouTubeClient.kt index ae12004f2..742ee5270 100644 --- a/innertube/src/main/java/com/zionhuang/innertube/models/YouTubeClient.kt +++ b/innertube/src/main/java/com/zionhuang/innertube/models/YouTubeClient.kt @@ -6,10 +6,13 @@ import kotlinx.serialization.Serializable data class YouTubeClient( val clientName: String, val clientVersion: String, - val api_key: String, + val clientId: String, val userAgent: String, val osVersion: String? = null, - val referer: String? = null, + val supportsLogin: Boolean = false, + val useSignatureTimestamp: Boolean = false, + // val origin: String? = null, + // val referer: String? = null, ) { fun toContext(locale: YouTubeLocale, visitorData: String?) = Context( client = Context.Client( @@ -23,54 +26,55 @@ data class YouTubeClient( ) companion object { - private const val REFERER_YOUTUBE_MUSIC = "https://music.youtube.com/" + /** + * Should be the latest Firefox ESR version. + */ + const val USER_AGENT_WEB = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0" - private const val USER_AGENT_WEB = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36" - private const val USER_AGENT_ANDROID = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Mobile Safari/537.36" - private const val USER_AGENT_IOS = "com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)" - - val ANDROID_MUSIC = YouTubeClient( - clientName = "ANDROID_MUSIC", - clientVersion = "5.01", - api_key = "AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI", - userAgent = USER_AGENT_ANDROID - ) - - val ANDROID = YouTubeClient( - clientName = "ANDROID", - clientVersion = "17.13.3", - api_key = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", - userAgent = USER_AGENT_ANDROID, - ) + const val ORIGIN_YOUTUBE_MUSIC = "https://music.youtube.com" + const val REFERER_YOUTUBE_MUSIC = "$ORIGIN_YOUTUBE_MUSIC/" + const val API_URL_YOUTUBE_MUSIC = "$ORIGIN_YOUTUBE_MUSIC/youtubei/v1/" val WEB = YouTubeClient( clientName = "WEB", - clientVersion = "2.2021111", - api_key = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX3", - userAgent = USER_AGENT_WEB + clientVersion = "2.20241126.01.00", + clientId = "1", + userAgent = USER_AGENT_WEB, ) val WEB_REMIX = YouTubeClient( clientName = "WEB_REMIX", - clientVersion = "1.20220606.03.00", - api_key = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", + clientVersion = "1.20241127.01.00", + clientId = "67", + userAgent = USER_AGENT_WEB, + supportsLogin = true, + useSignatureTimestamp = true, + ) + + val WEB_CREATOR = YouTubeClient( + clientName = "WEB_CREATOR", + clientVersion = "1.20241203.01.00", + clientId = "62", userAgent = USER_AGENT_WEB, - referer = REFERER_YOUTUBE_MUSIC + supportsLogin = true, + useSignatureTimestamp = true, ) val TVHTML5 = YouTubeClient( clientName = "TVHTML5_SIMPLY_EMBEDDED_PLAYER", clientVersion = "2.0", - api_key = "AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8", - userAgent = "Mozilla/5.0 (PlayStation 4 5.55) AppleWebKit/601.2 (KHTML, like Gecko)" + clientId = "85", + userAgent = "Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15", + supportsLogin = true, + useSignatureTimestamp = true, ) val IOS = YouTubeClient( clientName = "IOS", - clientVersion = "19.29.1", - api_key = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", - userAgent = USER_AGENT_IOS, - osVersion = "17.5.1.21F90", + clientVersion = "19.45.4", + clientId = "5", + userAgent = "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)", + osVersion = "18.1.0.22B83", ) } } diff --git a/innertube/src/main/java/com/zionhuang/innertube/models/body/PlayerBody.kt b/innertube/src/main/java/com/zionhuang/innertube/models/body/PlayerBody.kt index 3522294db..642d660ea 100644 --- a/innertube/src/main/java/com/zionhuang/innertube/models/body/PlayerBody.kt +++ b/innertube/src/main/java/com/zionhuang/innertube/models/body/PlayerBody.kt @@ -8,5 +8,17 @@ data class PlayerBody( val context: Context, val videoId: String, val playlistId: String?, + val playbackContext: PlaybackContext? = null, val contentCheckOk: Boolean = true, -) + val racyCheckOk: Boolean = true, +) { + @Serializable + data class PlaybackContext( + val contentPlaybackContext: ContentPlaybackContext + ) { + @Serializable + data class ContentPlaybackContext( + val signatureTimestamp: Int + ) + } +} diff --git a/innertube/src/main/java/com/zionhuang/innertube/models/response/PlayerResponse.kt b/innertube/src/main/java/com/zionhuang/innertube/models/response/PlayerResponse.kt index 23d426879..87394ee76 100644 --- a/innertube/src/main/java/com/zionhuang/innertube/models/response/PlayerResponse.kt +++ b/innertube/src/main/java/com/zionhuang/innertube/models/response/PlayerResponse.kt @@ -2,7 +2,10 @@ package com.zionhuang.innertube.models.response import com.zionhuang.innertube.models.ResponseContext import com.zionhuang.innertube.models.Thumbnails +import io.ktor.http.URLBuilder +import io.ktor.http.parseQueryString import kotlinx.serialization.Serializable +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager /** * PlayerResponse with [com.zionhuang.innertube.models.YouTubeClient.ANDROID_MUSIC] client @@ -57,9 +60,25 @@ data class PlayerResponse( val audioChannels: Int?, val loudnessDb: Double?, val lastModified: Long?, + val signatureCipher: String?, ) { val isAudio: Boolean get() = width == null + + fun findUrl(): String? { + this.url?.let { + return it + } + this.signatureCipher?.let { signatureCipher -> + val params = parseQueryString(signatureCipher) + val obfuscatedSignature = params["s"] ?: return null + val signatureParam = params["sp"] ?: return null + val url = params["url"]?.let { URLBuilder(it) } ?: return null + url.parameters[signatureParam] = YoutubeJavaScriptPlayerManager.deobfuscateSignature("", obfuscatedSignature) + return YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated("", url.toString()) + } + return null + } } }