Add support for 7zip compressed ROMs

buildSrc/src/main/java/deps.kt

@@ -126,6 +126,8 @@ object deps {
         const val material                 = "com.google.android.material:material:1.2.1"
         const val radialgamepad            = "com.github.Swordfish90:RadialGamePad:${versions.radialgamepad}"
         const val libretrodroid            = "com.github.Swordfish90:LibretroDroid:${versions.libretrodroid}"
+        const val xz                       = "org.tukaani:xz:1.8"
+        const val compress                 = "org.apache.commons:commons-compress:1.20"
 
         // This will be replaced by native material components when they will be ready.
         const val materialProgressBar      = "me.zhanghai.android.materialprogressbar:library:1.6.1"

retrograde-app-shared/build.gradle.kts

@@ -38,6 +38,8 @@ dependencies {
     implementation(deps.libs.rxKotlin2)
     implementation(deps.libs.rxRelay2)
     implementation(deps.libs.kotlin.serialization)
+    implementation(deps.libs.xz)
+    implementation(deps.libs.compress)
 
     kapt(deps.libs.androidx.room.compiler)
 }

retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/DocumentFileParser.kt

@@ -6,7 +6,11 @@ import com.swordfish.lemuroid.common.kotlin.toStringCRC32
 import com.swordfish.lemuroid.lib.storage.BaseStorageFile
 import com.swordfish.lemuroid.lib.storage.ISOScanner
 import com.swordfish.lemuroid.lib.storage.StorageFile
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry
+import org.apache.commons.compress.archivers.sevenz.SevenZFile
 import timber.log.Timber
+import java.io.File
+import java.io.InputStream
 import java.util.zip.ZipEntry
 import java.util.zip.ZipInputStream
 
@@ -17,11 +21,47 @@ object DocumentFileParser {
     private const val MAX_SIZE_CRC32 = 1_000_000_000
 
     fun parseDocumentFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile {
-        return if (baseStorageFile.extension == "zip") {
-            Timber.d("Detected zip file. ${baseStorageFile.name}")
-            parseZipFile(context, baseStorageFile)
+        return when (baseStorageFile.extension) {
+            "zip" -> {
+                Timber.d("Detected zip file. ${baseStorageFile.name}")
+                parseZipFile(context, baseStorageFile)
+            }
+
+            "7z" -> {
+                Timber.d("Detected 7z file. ${baseStorageFile.name}")
+                parseSevenZFile(context, baseStorageFile)
+            }
+
+            else -> {
+                Timber.d("Detected standard file. ${baseStorageFile.name}")
+                parseStandardFile(context, baseStorageFile)
+            }
+        }
+    }
+
+    private fun parseSevenZFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile {
+        /* Apache Compress' 7z implementation does not supports IO streams, but only File.
+         To create a SevenZFile from Android's Uri, we are bound to create a temp File.
+         Another option is to extract the exact path name and create a SevenZFile,
+         but this is quite inconsistent for different Android API versions.
+         */
+        val file = File.createTempFile("temp_seven_z_file_from_uri", ".7z", context.cacheDir)
+        val inputStream = context.contentResolver.openInputStream(baseStorageFile.uri)
+        inputStream?.let { file.copyInputStreamToFile(it) }
+        val sevenZFile = SevenZFile(file)
+        val gameEntry = findGameEntry(sevenZFile, baseStorageFile.size)
+        return if (gameEntry != null) {
+            Timber.d("Handing 7z file as compressed game: ${baseStorageFile.name}")
+            val entryInputStream = kotlin.runCatching {
+                sevenZFile.getInputStream(gameEntry)
+            }.getOrNull()
+            parseSevenZCompressedGame(
+                baseStorageFile,
+                gameEntry,
+                entryInputStream
+            ).also { sevenZFile.close() }
         } else {
-            Timber.d("Detected standard file. ${baseStorageFile.name}")
+            Timber.d("Handing 7z file as standard: ${baseStorageFile.name}")
             parseStandardFile(context, baseStorageFile)
         }
     }
@@ -59,6 +99,25 @@ object DocumentFileParser {
         )
     }
 
+    private fun parseSevenZCompressedGame(
+        baseStorageFile: BaseStorageFile,
+        entry: SevenZArchiveEntry,
+        inputStream: InputStream?
+    ): StorageFile {
+        Timber.d("Processing sevenZ entry: ${entry.name}")
+
+        val serial = inputStream?.let { ISOScanner.extractSerial(entry.name, it) }
+
+        return StorageFile(
+            entry.name,
+            entry.size,
+            entry.crcValue.toStringCRC32(),
+            serial,
+            baseStorageFile.uri,
+            baseStorageFile.uri.path
+        )
+    }
+
     private fun parseStandardFile(context: Context, baseStorageFile: BaseStorageFile): StorageFile {
         val serial = context.contentResolver.openInputStream(baseStorageFile.uri)
             ?.let { inputStream -> ISOScanner.extractSerial(baseStorageFile.name, inputStream) }
@@ -94,8 +153,43 @@ object DocumentFileParser {
         return null
     }
 
+    /* Finds a sevenZ entry which we assume is a game. Lemuroids only supports single archive games,
+       so we are looking for an entry which occupies a large percentage of the archive space.
+       This is very fast heuristic to compute and avoids reading the whole stream in most
+       scenarios.*/
+    fun findGameEntry(sevenZFile: SevenZFile, fileSize: Long = -1): SevenZArchiveEntry? {
+        for (i in 0..MAX_CHECKED_ENTRIES) {
+            val entry = sevenZFile.nextEntry ?: break
+            if (!isGameEntry(entry, fileSize)) continue
+            return entry
+        }
+        sevenZFile.close()
+        return null
+    }
+
     private fun isGameEntry(entry: ZipEntry, fileSize: Long): Boolean {
         if (fileSize <= 0 || entry.compressedSize <= 0) return false
         return (entry.compressedSize.toFloat() / fileSize.toFloat()) > SINGLE_ARCHIVE_THRESHOLD
     }
+
+    private fun isGameEntry(entry: SevenZArchiveEntry, fileSize: Long): Boolean {
+        if (fileSize <= 0 || entry.size <= 0) return false
+        return (entry.size.toFloat() / fileSize.toFloat()) > SINGLE_ARCHIVE_THRESHOLD
+    }
+
+    private fun File.copyInputStreamToFile(inputStream: InputStream) {
+        val buffer = ByteArray(1024)
+
+        inputStream.use { input ->
+            this.outputStream().use { fileOut ->
+                while (true) {
+                    val length = input.read(buffer)
+                    if (length <= 0)
+                        break
+                    fileOut.write(buffer, 0, length)
+                }
+                fileOut.flush()
+            }
+        }
+    }
 }

retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/LocalStorageProvider.kt

@@ -25,6 +25,7 @@ import androidx.core.net.toUri
 import androidx.leanback.preference.LeanbackPreferenceFragment
 import androidx.preference.PreferenceManager
 import com.swordfish.lemuroid.common.kotlin.extractEntryToFile
+import com.swordfish.lemuroid.common.kotlin.isSevenZipped
 import com.swordfish.lemuroid.common.kotlin.isZipped
 import com.swordfish.lemuroid.lib.R
 import com.swordfish.lemuroid.lib.library.db.entity.DataFile
@@ -37,6 +38,7 @@ import com.swordfish.lemuroid.lib.storage.StorageProvider
 import io.reactivex.Completable
 import io.reactivex.Observable
 import io.reactivex.Single
+import org.apache.commons.compress.archivers.sevenz.SevenZFile
 import java.io.File
 import java.io.InputStream
 import java.util.zip.ZipInputStream
@@ -110,6 +112,11 @@ class LocalStorageProvider(
             stream.extractEntryToFile(game.fileName, cacheFile)
         }
 
+        if(originalFile.isSevenZipped()) {
+            val sevenZFile = SevenZFile(originalFile)
+            sevenZFile.extractEntryToFile(game.fileName, cacheFile)
+        }
+
         cacheFile
     }
 

retrograde-app-shared/src/main/java/com/swordfish/lemuroid/lib/storage/local/StorageAccessFrameworkProvider.kt

@@ -8,7 +8,9 @@ import androidx.leanback.preference.LeanbackPreferenceFragment
 import androidx.preference.PreferenceManager
 import com.swordfish.lemuroid.common.kotlin.extractEntryToFile
 import com.swordfish.lemuroid.common.kotlin.isZipped
+import com.swordfish.lemuroid.common.kotlin.isSevenZipped
 import com.swordfish.lemuroid.common.kotlin.writeToFile
+import com.swordfish.lemuroid.common.kotlin.copyInputStreamToFile
 import com.swordfish.lemuroid.lib.R
 import com.swordfish.lemuroid.lib.library.db.entity.DataFile
 import com.swordfish.lemuroid.lib.library.db.entity.Game
@@ -19,6 +21,7 @@ import com.swordfish.lemuroid.lib.storage.StorageProvider
 import io.reactivex.Completable
 import io.reactivex.Observable
 import io.reactivex.Single
+import org.apache.commons.compress.archivers.sevenz.SevenZFile
 import timber.log.Timber
 import java.io.File
 import java.io.InputStream
@@ -161,6 +164,12 @@ class StorageAccessFrameworkProvider(
                 context.contentResolver.openInputStream(originalDocument.uri)
             )
             stream.extractEntryToFile(game.fileName, cacheFile)
+        } else if (originalDocument.isSevenZipped() && originalDocument.name != game.fileName) {
+            val file = File.createTempFile("temp_seven_z_file_from_uri", ".7z", context.cacheDir)
+            val inputStream = context.contentResolver.openInputStream(originalDocument.uri)
+            inputStream?.let { file.copyInputStreamToFile(it) }
+            val sevenZFile = SevenZFile(file)
+            sevenZFile.extractEntryToFile(game.fileName, cacheFile)
         } else {
             val stream = context.contentResolver.openInputStream(originalDocument.uri)!!
             stream.writeToFile(cacheFile)

retrograde-util/build.gradle.kts

@@ -14,4 +14,6 @@ dependencies {
     implementation(deps.libs.koptionalRxJava2)
     implementation(deps.libs.okHttp3)
     implementation(deps.libs.rxJava2)
+    implementation(deps.libs.xz)
+    implementation(deps.libs.compress)
 }

retrograde-util/src/main/java/com/swordfish/lemuroid/common/kotlin/FileKt.kt

@@ -20,9 +20,12 @@
 package com.swordfish.lemuroid.common.kotlin
 
 import androidx.documentfile.provider.DocumentFile
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry
+import org.apache.commons.compress.archivers.sevenz.SevenZFile
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.File
+import java.io.FileOutputStream
 import java.io.InputStream
 import java.io.PushbackInputStream
 import java.util.zip.CRC32
@@ -62,10 +65,29 @@ fun ZipInputStream.extractEntryToFile(entryName: String, gameFile: File) {
     }
 }
 
+fun SevenZFile.extractEntryToFile(entryName: String, gameFile: File) {
+    var entry: SevenZArchiveEntry
+    while (this.nextEntry.also { entry = it } != null) {
+        if (entryName == gameFile.name) {
+            val out = FileOutputStream(gameFile)
+            val content = ByteArray(entry.size.toInt())
+            this.read(content, 0, content.size)
+            out.write(content)
+            out.close()
+            break
+        }
+    }
+    this.close()
+}
+
 fun File.isZipped() = extension == "zip"
 
+fun File.isSevenZipped() = extension == "7z"
+
 fun DocumentFile.isZipped() = type == "application/zip"
 
+fun DocumentFile.isSevenZipped() = type == "application/x-7z-compressed"
+
 /** Returns the uncompressed input stream if gzip compressed. */
 private fun File.uncompressedInputStream(): InputStream {
     val pb = PushbackInputStream(inputStream(), 2)
@@ -76,6 +98,22 @@ private fun File.uncompressedInputStream(): InputStream {
         GZIPInputStream(pb, GZIP_INPUT_STREAM_BUFFER_SIZE) else pb
 }
 
+fun File.copyInputStreamToFile(inputStream: InputStream) {
+    val buffer = ByteArray(1024)
+
+    inputStream.use { input ->
+        this.outputStream().use { fileOut ->
+            while (true) {
+                val length = input.read(buffer)
+                if (length <= 0)
+                    break
+                fileOut.write(buffer, 0, length)
+            }
+            fileOut.flush()
+        }
+    }
+}
+
 /** Write bytes to file using GZIP compression. */
 fun File.writeBytesCompressed(array: ByteArray) {
     val inputStream = ByteArrayInputStream(array)