Skip to content

Commit 34b9547

Browse files
nicobossnicoboss
authored
Merge pull request #127 from nicoboss/zstd-long-distance-mode
Zstd long distance mode
2 parents 2d3ab4b + a5a1602 commit 34b9547

File tree

5 files changed

+39
-33
lines changed

5 files changed

+39
-33
lines changed

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ Use the following command to install the GUI versions requirements:\
3333
## Usage
3434
```
3535
nsz --help
36-
usage: nsz.py [-h] [-C] [-D] [-l LEVEL] [-B] [-S] [-s BS] [-V] [-p] [-P]
36+
usage: nsz.py [-h] [-C] [-D] [-l LEVEL] [-L] [-B] [-S] [-s BS] [-V] [-p] [-P]
3737
[-t THREADS] [-m MULTI] [-o [OUTPUT]] [-w] [-r] [--rm-source]
3838
[-i] [--depth DEPTH] [-x] [--extractregex EXTRACTREGEX]
3939
[--titlekeys] [--undupe] [--undupe-dryrun] [--undupe-rename]
@@ -46,13 +46,15 @@ usage: nsz.py [-h] [-C] [-D] [-l LEVEL] [-B] [-S] [-s BS] [-V] [-p] [-P]
4646
positional arguments:
4747
file
4848
49-
options:
49+
optional arguments:
5050
-h, --help show this help message and exit
5151
-C Compress NSP/XCI
5252
-D Decompress NSZ/XCZ/NCZ
5353
-l LEVEL, --level LEVEL
5454
Compression Level: Trade-off between compression speed
5555
and compression ratio. Default: 18, Max: 22
56+
-L, --long Enables zStandard long distance mode for even better
57+
compression
5658
-B, --block Use block compression option. This mode allows highly
5759
multi-threaded compression/decompression with random
5860
read access allowing compressed games to be played

‎nsz/BlockCompressor.py‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from time import sleep
44
from pathlib import Path
55
from traceback import format_exc
6-
from zstandard import ZstdCompressor
6+
from zstandard import ZstdCompressionParameters, ZstdCompressor
77
from nsz.ThreadSafeCounter import Counter
88
from nsz.SectionFs import isNcaPacked, sortedFs
99
from multiprocessing import Process, Manager
@@ -19,22 +19,23 @@ def compressBlockTask(in_queue, out_list, readyForWork, pleaseKillYourself, bloc
1919
#readyForWork.decrement() # https://github.com/nicoboss/nsz/issues/80
2020
if pleaseKillYourself.value() > 0:
2121
break
22-
buffer, compressionLevel, compressedblockSizeList, chunkRelativeBlockID = item # compressedblockSizeList IS UNUSED VARIABLE
22+
buffer, compressionLevel, useLongDistanceMode, compressedblockSizeList, chunkRelativeBlockID = item # compressedblockSizeList IS UNUSED VARIABLE
2323
if buffer == 0:
2424
return
2525
if compressionLevel == 0 and len(buffer) == blockSize: # https://github.com/nicoboss/nsz/issues/79
2626
out_list[chunkRelativeBlockID] = buffer
2727
else:
28-
compressed = ZstdCompressor(level=compressionLevel).compress(buffer)
28+
params = ZstdCompressionParameters.from_level(compressionLevel, enable_ldm=useLongDistanceMode)
29+
compressed = ZstdCompressor(compression_params=params).compress(buffer)
2930
out_list[chunkRelativeBlockID] = compressed if len(compressed) < len(buffer) else buffer
3031

31-
def blockCompress(filePath, compressionLevel, blockSizeExponent, outputDir, threads):
32+
def blockCompress(filePath, compressionLevel, useLongDistanceMode, blockSizeExponent, outputDir, threads):
3233
if filePath.suffix == '.nsp':
33-
return blockCompressNsp(filePath, compressionLevel, blockSizeExponent, outputDir, threads)
34+
return blockCompressNsp(filePath, compressionLevel, useLongDistanceMode, blockSizeExponent, outputDir, threads)
3435
elif filePath.suffix == '.xci':
35-
return blockCompressXci(filePath, compressionLevel, blockSizeExponent, outputDir, threads)
36+
return blockCompressXci(filePath, compressionLevel, useLongDistanceMode, blockSizeExponent, outputDir, threads)
3637

37-
def blockCompressContainer(readContainer, writeContainer, compressionLevel, blockSizeExponent, threads):
38+
def blockCompressContainer(readContainer, writeContainer, compressionLevel, useLongDistanceMode, blockSizeExponent, threads):
3839
CHUNK_SZ = 0x100000
3940
UNCOMPRESSABLE_HEADER_SIZE = 0x4000
4041
if blockSizeExponent < 14 or blockSizeExponent > 32:
@@ -153,7 +154,7 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, bloc
153154
break
154155
chunkRelativeBlockID = 0
155156
startChunkBlockID = blockID
156-
work.put([buffer, compressionLevel, compressedblockSizeList, chunkRelativeBlockID])
157+
work.put([buffer, compressionLevel, useLongDistanceMode,compressedblockSizeList, chunkRelativeBlockID])
157158
readyForWork.decrement()
158159
blockID += 1
159160
chunkRelativeBlockID += 1
@@ -202,17 +203,17 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, bloc
202203
sleep(0.02)
203204

204205

205-
def blockCompressNsp(filePath, compressionLevel , blockSizeExponent, outputDir, threads):
206+
def blockCompressNsp(filePath, compressionLevel, useLongDistanceMode, blockSizeExponent, outputDir, threads):
206207
filePath = filePath.resolve()
207208
container = factory(filePath)
208209
container.open(str(filePath), 'rb')
209210
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
210211

211-
Print.info('Block compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, nszPath))
212+
Print.info(f'Block compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}')
212213

213214
try:
214215
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
215-
blockCompressContainer(container, nsp, compressionLevel, blockSizeExponent, threads)
216+
blockCompressContainer(container, nsp, compressionLevel, useLongDistanceMode, blockSizeExponent, threads)
216217
except BaseException as ex:
217218
if not ex is KeyboardInterrupt:
218219
Print.error(format_exc())
@@ -222,19 +223,19 @@ def blockCompressNsp(filePath, compressionLevel , blockSizeExponent, outputDir,
222223
container.close()
223224
return nszPath
224225

225-
def blockCompressXci(filePath, compressionLevel, blockSizeExponent, outputDir, threads):
226+
def blockCompressXci(filePath, compressionLevel, useLongDistanceMode, blockSizeExponent, outputDir, threads):
226227
filePath = filePath.resolve()
227228
container = factory(filePath)
228229
container.open(str(filePath), 'rb')
229230
secureIn = container.hfs0['secure']
230231
xczPath = outputDir.joinpath(filePath.stem + '.xcz')
231232

232-
Print.info('Block compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, xczPath))
233+
Print.info(f'Block compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {xczPath}')
233234

234235
try:
235236
with Xci.XciStream(str(xczPath), originalXciPath = filePath) as xci: # need filepath to copy XCI container settings
236237
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0), xci.f.tell()) as secureOut:
237-
blockCompressContainer(secureIn, secureOut, compressionLevel, blockSizeExponent, threads)
238+
blockCompressContainer(secureIn, secureOut, compressionLevel, useLongDistanceMode, blockSizeExponent, threads)
238239

239240
xci.hfs0.resize('secure', secureOut.actualSize)
240241
except BaseException as ex:

‎nsz/ParseArguments.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ def parse():
88
parser.add_argument('-C', action="store_true", help='Compress NSP/XCI')
99
parser.add_argument('-D', action="store_true", help='Decompress NSZ/XCZ/NCZ')
1010
parser.add_argument('-l', '--level', type=int, default=18, help='Compression Level: Trade-off between compression speed and compression ratio. Default: 18, Max: 22')
11+
parser.add_argument('-L', '--long', action="store_true", default=False, help='Enables zStandard long distance mode for even better compression')
1112
parser.add_argument('-B', '--block', action="store_true", default=False, help="Use block compression option. This mode allows highly multi-threaded compression/decompression with random read access allowing compressed games to be played without decompression in the future however this comes with a slightly lower compression ratio cost. This is the default option for XCZ.")
1213
parser.add_argument('-S', '--solid', action="store_true", default=False, help="Use solid compression option. Slightly higher compression ratio but won't allow for random read access. File compressed this way will never be mountable (have to be installed or decompressed first to run). This is the default option for NSZ.")
1314
parser.add_argument('-s', '--bs', type=int, default=20, help='Block Size for random read access 2^x while x between 14 and 32. Default: 20 => 1 MB')

‎nsz/SolidCompressor.py‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,20 @@
44
from traceback import format_exc
55
from nsz.SectionFs import isNcaPacked, sortedFs
66
from nsz.Fs import factory, Ticket, Pfs0, Hfs0, Nca, Type, Xci
7-
from zstandard import FLUSH_FRAME, ZstdCompressor
7+
from zstandard import FLUSH_FRAME, COMPRESSOBJ_FLUSH_FINISH, ZstdCompressionParameters, ZstdCompressor
88
from nsz.PathTools import *
99

1010
UNCOMPRESSABLE_HEADER_SIZE = 0x4000
1111
CHUNK_SZ = 0x1000000
1212

1313

14-
def solidCompress(filePath, compressionLevel, outputDir, threads, statusReport, id, pleaseNoPrint):
14+
def solidCompress(filePath, compressionLevel, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint):
1515
if filePath.suffix == '.nsp':
16-
return solidCompressNsp(filePath, compressionLevel, outputDir, threads, statusReport, id, pleaseNoPrint)
16+
return solidCompressNsp(filePath, compressionLevel, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint)
1717
elif filePath.suffix == '.xci':
18-
return solidCompressXci(filePath, compressionLevel, outputDir, threads, statusReport, id, pleaseNoPrint)
18+
return solidCompressXci(filePath, compressionLevel, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint)
1919

20-
def processContainer(readContainer, writeContainer, compressionLevel, threads, statusReport, id, pleaseNoPrint):
20+
def processContainer(readContainer, writeContainer, compressionLevel, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint):
2121
for nspf in readContainer:
2222
if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
2323
Print.info('[SKIPPED] Delta fragment {0}'.format(nspf._path), pleaseNoPrint)
@@ -82,9 +82,11 @@ def processContainer(readContainer, writeContainer, compressionLevel, threads, s
8282
partNr = 0
8383
statusReport[id] = [nspf.tell(), f.tell(), nspf.size, 'Compressing']
8484
if threads > 1:
85-
cctx = ZstdCompressor(level=compressionLevel, threads=threads)
85+
params = ZstdCompressionParameters.from_level(compressionLevel, enable_ldm=useLongDistanceMode, threads=threads)
86+
cctx = ZstdCompressor(compression_params=params)
8687
else:
87-
cctx = ZstdCompressor(level=compressionLevel)
88+
params = ZstdCompressionParameters.from_level(compressionLevel, enable_ldm=useLongDistanceMode)
89+
cctx = ZstdCompressor(compression_params=params)
8890
compressor = cctx.stream_writer(f)
8991
while True:
9092

@@ -120,17 +122,17 @@ def processContainer(readContainer, writeContainer, compressionLevel, threads, s
120122
f.write(buffer)
121123

122124

123-
def solidCompressNsp(filePath, compressionLevel, outputDir, threads, statusReport, id, pleaseNoPrint):
125+
def solidCompressNsp(filePath, compressionLevel, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint):
124126
filePath = filePath.resolve()
125127
container = factory(filePath)
126128
container.open(str(filePath), 'rb')
127129
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
128130

129-
Print.info('Solid compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, nszPath), pleaseNoPrint)
131+
Print.info(f'Solid compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}', pleaseNoPrint)
130132

131133
try:
132134
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
133-
processContainer(container, nsp, compressionLevel, threads, statusReport, id, pleaseNoPrint)
135+
processContainer(container, nsp, compressionLevel, useLongDistanceMode,threads, statusReport, id, pleaseNoPrint)
134136
except BaseException as ex:
135137
if not ex is KeyboardInterrupt:
136138
Print.error(format_exc())
@@ -140,19 +142,19 @@ def solidCompressNsp(filePath, compressionLevel, outputDir, threads, statusRepor
140142
container.close()
141143
return nszPath
142144

143-
def solidCompressXci(filePath, compressionLevel, outputDir, threads, statusReport, id, pleaseNoPrint):
145+
def solidCompressXci(filePath, compressionLevel, useLongDistanceMode, outputDir, threads, statusReport, id, pleaseNoPrint):
144146
filePath = filePath.resolve()
145147
container = factory(filePath)
146148
container.open(str(filePath), 'rb')
147149
secureIn = container.hfs0['secure']
148150
xczPath = outputDir.joinpath(filePath.stem + '.xcz')
149151

150-
Print.info('Solid compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, xczPath), pleaseNoPrint)
152+
Print.info(f'Solid compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {xczPath}', pleaseNoPrint)
151153

152154
try:
153155
with Xci.XciStream(str(xczPath), originalXciPath = filePath) as xci: # need filepath to copy XCI container settings
154156
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint), xci.f.tell()) as secureOut:
155-
processContainer(secureIn, secureOut, compressionLevel, threads, statusReport, id, pleaseNoPrint)
157+
processContainer(secureIn, secureOut, compressionLevel, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint)
156158

157159
xci.hfs0.resize('secure', secureOut.actualSize)
158160
except BaseException as ex:

‎nsz/__init__.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ def solidCompressTask(in_queue, statusReport, readyForWork, pleaseNoPrint, pleas
3232
if pleaseKillYourself.value() > 0:
3333
break
3434
try:
35-
filePath, compressionLevel, outputDir, threadsToUse, verifyArg = item
36-
outFile = solidCompress(filePath, compressionLevel, outputDir, threadsToUse, statusReport, id, pleaseNoPrint)
35+
filePath, compressionLevel, useLongDistanceMode, outputDir, threadsToUse, verifyArg = item
36+
outFile = solidCompress(filePath, compressionLevel, useLongDistanceMode, outputDir, threadsToUse, statusReport, id, pleaseNoPrint)
3737
if verifyArg:
3838
Print.info("[VERIFY NSZ] {0}".format(outFile))
3939
try:
@@ -53,13 +53,13 @@ def compress(filePath, outputDir, args, work, amountOfTastkQueued):
5353

5454
if filePath.suffix == ".xci" and not args.solid or args.block:
5555
threadsToUseForBlockCompression = args.threads if args.threads > 0 else cpu_count()
56-
outFile = blockCompress(filePath, compressionLevel, args.bs, outputDir, threadsToUseForBlockCompression)
56+
outFile = blockCompress(filePath, compressionLevel, args.long, args.bs, outputDir, threadsToUseForBlockCompression)
5757
if args.verify:
5858
Print.info("[VERIFY NSZ] {0}".format(outFile))
5959
verify(outFile, True)
6060
else:
6161
threadsToUseForSolidCompression = args.threads if args.threads > 0 else 3
62-
work.put([filePath, compressionLevel, outputDir, threadsToUseForSolidCompression, args.verify])
62+
work.put([filePath, compressionLevel, args.long, outputDir, threadsToUseForSolidCompression, args.verify])
6363
amountOfTastkQueued.increment()
6464

6565

0 commit comments

Comments
 (0)