From 9c4818045281fc2960e34a0fa3ca51aa8e0f3955 Mon Sep 17 00:00:00 2001 From: Ashish Ekka Date: Fri, 27 Nov 2020 16:15:48 +0530 Subject: [PATCH] Add support for 7zip compressed ROMs --- buildSrc/src/main/java/deps.kt | 2 + retrograde-app-shared/build.gradle.kts | 2 + .../lib/storage/local/DocumentFileParser.kt | 102 +++++++++++++++++- .../lib/storage/local/LocalStorageProvider.kt | 7 ++ .../local/StorageAccessFrameworkProvider.kt | 9 ++ retrograde-util/build.gradle.kts | 2 + .../lemuroid/common/kotlin/FileKt.kt | 38 +++++++ 7 files changed, 158 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/java/deps.kt b/buildSrc/src/main/java/deps.kt index 565627e66a..0b63cde7a1 100644 --- a/buildSrc/src/main/java/deps.kt +++ b/buildSrc/src/main/java/deps.kt @@ -126,6 +126,8 @@ object deps { const val material = "com.google.android.material:material:1.2.1" const val radialgamepad = "com.github.Swordfish90:RadialGamePad:${versions.radialgamepad}" const val libretrodroid = "com.github.Swordfish90:LibretroDroid:${versions.libretrodroid}" + const val xz = "org.tukaani:xz:1.8" + const val compress = "org.apache.commons:commons-compress:1.20" // This will be replaced by native material components when they will be ready. const val materialProgressBar = "me.zhanghai.android.materialprogressbar:library:1.6.1" diff --git a/retrograde-app-shared/build.gradle.kts b/retrograde-app-shared/build.gradle.kts index 4b0861545f..98067c2edb 100644 --- a/retrograde-app-shared/build.gradle.kts +++ b/retrograde-app-shared/build.gradle.kts @@ -38,6 +38,8 @@ dependencies { implementation(deps.libs.rxKotlin2) implementation(deps.libs.rxRelay2) implementation(deps.libs.kotlin.serialization) + implementation(deps.libs.xz) + implementation(deps.libs.compress) kapt(deps.libs.androidx.room.compiler) } diff --git a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/DocumentFileParser.kt b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/DocumentFileParser.kt index 576f2d4d53..f4bad72230 100644 --- a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/DocumentFileParser.kt +++ b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/DocumentFileParser.kt @@ -6,7 +6,11 @@ import com.swordfish.lemuroid.common.kotlin.toStringCRC32 import com.swordfish.lemuroid.lib.storage.BaseStorageFile import com.swordfish.lemuroid.lib.storage.ISOScanner import com.swordfish.lemuroid.lib.storage.StorageFile +import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry +import org.apache.commons.compress.archivers.sevenz.SevenZFile import timber.log.Timber +import java.io.File +import java.io.InputStream import java.util.zip.ZipEntry import java.util.zip.ZipInputStream @@ -17,11 +21,47 @@ object DocumentFileParser { private const val MAX_SIZE_CRC32 = 1_000_000_000 fun parseDocumentFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile { - return if (baseStorageFile.extension == "zip") { - Timber.d("Detected zip file. ${baseStorageFile.name}") - parseZipFile(context, baseStorageFile) + return when (baseStorageFile.extension) { + "zip" -> { + Timber.d("Detected zip file. ${baseStorageFile.name}") + parseZipFile(context, baseStorageFile) + } + + "7z" -> { + Timber.d("Detected 7z file. ${baseStorageFile.name}") + parseSevenZFile(context, baseStorageFile) + } + + else -> { + Timber.d("Detected standard file. ${baseStorageFile.name}") + parseStandardFile(context, baseStorageFile) + } + } + } + + private fun parseSevenZFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile { + /* Apache Compress' 7z implementation does not supports IO streams, but only File. + To create a SevenZFile from Android's Uri, we are bound to create a temp File. + Another option is to extract the exact path name and create a SevenZFile, + but this is quite inconsistent for different Android API versions. + */ + val file = File.createTempFile("temp_seven_z_file_from_uri", ".7z", context.cacheDir) + val inputStream = context.contentResolver.openInputStream(baseStorageFile.uri) + inputStream?.let { file.copyInputStreamToFile(it) } + val sevenZFile = SevenZFile(file) + val gameEntry = findGameEntry(sevenZFile, baseStorageFile.size) + return if (gameEntry != null) { + Timber.d("Handing 7z file as compressed game: ${baseStorageFile.name}") + val entryInputStream = kotlin.runCatching { + sevenZFile.getInputStream(gameEntry) + }.getOrNull() + parseSevenZCompressedGame( + baseStorageFile, + gameEntry, + entryInputStream + ).also { sevenZFile.close() } } else { - Timber.d("Detected standard file. ${baseStorageFile.name}") + Timber.d("Handing 7z file as standard: ${baseStorageFile.name}") parseStandardFile(context, baseStorageFile) } } @@ -59,6 +99,25 @@ object DocumentFileParser { ) } + private fun parseSevenZCompressedGame( + baseStorageFile: BaseStorageFile, + entry: SevenZArchiveEntry, + inputStream: InputStream? + ): StorageFile { + Timber.d("Processing sevenZ entry: ${entry.name}") + + val serial = inputStream?.let { ISOScanner.extractSerial(entry.name, it) } + + return StorageFile( + entry.name, + entry.size, + entry.crcValue.toStringCRC32(), + serial, + baseStorageFile.uri, + baseStorageFile.uri.path + ) + } + private fun parseStandardFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile { val serial = context.contentResolver.openInputStream(baseStorageFile.uri) ?.let { inputStream -> ISOScanner.extractSerial(baseStorageFile.name, inputStream) } @@ -94,8 +153,43 @@ object DocumentFileParser { return null } + /* Finds a sevenZ entry which we assume is a game. Lemuroids only supports single archive games, + so we are looking for an entry which occupies a large percentage of the archive space. + This is very fast heuristic to compute and avoids reading the whole stream in most + scenarios.*/ + fun findGameEntry(sevenZFile: SevenZFile, fileSize: Long = -1): SevenZArchiveEntry? { + for (i in 0..MAX_CHECKED_ENTRIES) { + val entry = sevenZFile.nextEntry ?: break + if (!isGameEntry(entry, fileSize)) continue + return entry + } + sevenZFile.close() + return null + } + private fun isGameEntry(entry: ZipEntry, fileSize: Long): Boolean { if (fileSize <= 0 || entry.compressedSize <= 0) return false return (entry.compressedSize.toFloat() / fileSize.toFloat()) > SINGLE_ARCHIVE_THRESHOLD } + + private fun isGameEntry(entry: SevenZArchiveEntry, fileSize: Long): Boolean { + if (fileSize <= 0 || entry.size <= 0) return false + return (entry.size.toFloat() / fileSize.toFloat()) > SINGLE_ARCHIVE_THRESHOLD + } + + private fun File.copyInputStreamToFile(inputStream: InputStream) { + val buffer = ByteArray(1024) + + inputStream.use { input -> + this.outputStream().use { fileOut -> + while (true) { + val length = input.read(buffer) + if (length <= 0) + break + fileOut.write(buffer, 0, length) + } + fileOut.flush() + } + } + } } diff --git a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/LocalStorageProvider.kt b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/LocalStorageProvider.kt index 8295f73b1d..f8e44cabc8 100644 --- a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/LocalStorageProvider.kt +++ b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/LocalStorageProvider.kt @@ -25,6 +25,7 @@ import androidx.core.net.toUri import androidx.leanback.preference.LeanbackPreferenceFragment import androidx.preference.PreferenceManager import com.swordfish.lemuroid.common.kotlin.extractEntryToFile +import com.swordfish.lemuroid.common.kotlin.isSevenZipped import com.swordfish.lemuroid.common.kotlin.isZipped import com.swordfish.lemuroid.lib.R import com.swordfish.lemuroid.lib.library.db.entity.DataFile @@ -37,6 +38,7 @@ import com.swordfish.lemuroid.lib.storage.StorageProvider import io.reactivex.Completable import io.reactivex.Observable import io.reactivex.Single +import org.apache.commons.compress.archivers.sevenz.SevenZFile import java.io.File import java.io.InputStream import java.util.zip.ZipInputStream @@ -110,6 +112,11 @@ class LocalStorageProvider( stream.extractEntryToFile(game.fileName, cacheFile) } + if (originalFile.isSevenZipped()) { + val sevenZFile = SevenZFile(originalFile) + sevenZFile.extractEntryToFile(game.fileName, cacheFile) + } + cacheFile } diff --git a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/StorageAccessFrameworkProvider.kt b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/StorageAccessFrameworkProvider.kt index 5734900426..5fcfb407ff 100644 --- a/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/StorageAccessFrameworkProvider.kt +++ b/retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/StorageAccessFrameworkProvider.kt @@ -8,7 +8,9 @@ import androidx.leanback.preference.LeanbackPreferenceFragment import androidx.preference.PreferenceManager import com.swordfish.lemuroid.common.kotlin.extractEntryToFile import com.swordfish.lemuroid.common.kotlin.isZipped +import com.swordfish.lemuroid.common.kotlin.isSevenZipped import com.swordfish.lemuroid.common.kotlin.writeToFile +import com.swordfish.lemuroid.common.kotlin.copyInputStreamToFile import com.swordfish.lemuroid.lib.R import com.swordfish.lemuroid.lib.library.db.entity.DataFile import com.swordfish.lemuroid.lib.library.db.entity.Game @@ -19,6 +21,7 @@ import com.swordfish.lemuroid.lib.storage.StorageProvider import io.reactivex.Completable import io.reactivex.Observable import io.reactivex.Single +import org.apache.commons.compress.archivers.sevenz.SevenZFile import timber.log.Timber import java.io.File import java.io.InputStream @@ -161,6 +164,12 @@ class StorageAccessFrameworkProvider( context.contentResolver.openInputStream(originalDocument.uri) ) stream.extractEntryToFile(game.fileName, cacheFile) + } else if (originalDocument.isSevenZipped() && originalDocument.name != game.fileName) { + val file = File.createTempFile("temp_seven_z_file_from_uri", ".7z", context.cacheDir) + val inputStream = context.contentResolver.openInputStream(originalDocument.uri) + inputStream?.let { file.copyInputStreamToFile(it) } + val sevenZFile = SevenZFile(file) + sevenZFile.extractEntryToFile(game.fileName, cacheFile) } else { val stream = context.contentResolver.openInputStream(originalDocument.uri)!! stream.writeToFile(cacheFile) diff --git a/retrograde-util/build.gradle.kts b/retrograde-util/build.gradle.kts index 326aeb53f5..dc06090d32 100644 --- a/retrograde-util/build.gradle.kts +++ b/retrograde-util/build.gradle.kts @@ -14,4 +14,6 @@ dependencies { implementation(deps.libs.koptionalRxJava2) implementation(deps.libs.okHttp3) implementation(deps.libs.rxJava2) + implementation(deps.libs.xz) + implementation(deps.libs.compress) } diff --git a/retrograde-util/src/main/java/com/swordfish/lemuroid/common/kotlin/FileKt.kt b/retrograde-util/src/main/java/com/swordfish/lemuroid/common/kotlin/FileKt.kt index 35c702bb89..3c0d1e2daf 100644 --- a/retrograde-util/src/main/java/com/swordfish/lemuroid/common/kotlin/FileKt.kt +++ b/retrograde-util/src/main/java/com/swordfish/lemuroid/common/kotlin/FileKt.kt @@ -20,9 +20,12 @@ package com.swordfish.lemuroid.common.kotlin import androidx.documentfile.provider.DocumentFile +import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry +import org.apache.commons.compress.archivers.sevenz.SevenZFile import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File +import java.io.FileOutputStream import java.io.InputStream import java.io.PushbackInputStream import java.util.zip.CRC32 @@ -62,10 +65,29 @@ fun ZipInputStream.extractEntryToFile(entryName: String, gameFile: File) { } } +fun SevenZFile.extractEntryToFile(entryName: String, gameFile: File) { + var entry: SevenZArchiveEntry + while (this.nextEntry.also { entry = it } != null) { + if (entryName == gameFile.name) { + val out = FileOutputStream(gameFile) + val content = ByteArray(entry.size.toInt()) + this.read(content, 0, content.size) + out.write(content) + out.close() + break + } + } + this.close() +} + fun File.isZipped() = extension == "zip" +fun File.isSevenZipped() = extension == "7z" + fun DocumentFile.isZipped() = type == "application/zip" +fun DocumentFile.isSevenZipped() = type == "application/x-7z-compressed" + /** Returns the uncompressed input stream if gzip compressed. */ private fun File.uncompressedInputStream(): InputStream { val pb = PushbackInputStream(inputStream(), 2) @@ -76,6 +98,22 @@ private fun File.uncompressedInputStream(): InputStream { GZIPInputStream(pb, GZIP_INPUT_STREAM_BUFFER_SIZE) else pb } +fun File.copyInputStreamToFile(inputStream: InputStream) { + val buffer = ByteArray(1024) + + inputStream.use { input -> + this.outputStream().use { fileOut -> + while (true) { + val length = input.read(buffer) + if (length <= 0) + break + fileOut.write(buffer, 0, length) + } + fileOut.flush() + } + } +} + /** Write bytes to file using GZIP compression. */ fun File.writeBytesCompressed(array: ByteArray) { val inputStream = ByteArrayInputStream(array)