Merge pull request #89 from jakcron/v1.7-development

Update NSTool to v1.7.0

README.md

@@ -143,6 +143,16 @@ nstool -x /path/to/a/file.bin ./extract_dir/different_name.bin some_file.bin
 * NSP
 * XCI
 
+## NCA Patches
+Nintendo distributes game patches/updates in the style of a diff to keep file sizes down. This means extracting game patches requires the base version of the game to be able to process patch data. Typically this is only done for the Program NCA.
+
+If `basegame_v0.nca` is the base Program NCA, and `gamepatch_v13219.nca` is the patch Program NCA, simply specify the base NCA using the base NCA option `--basenca` when processing the patch NCA.
+
+```
+nstool --basenca ./basegame_v0.nca -x ./patchdata gamepatch_v13219.nca
+```
+In the above example the patch NCA is being extracted to `./patchdata`
+
 ## Encrypted Files
 Some Nintendo Switch files are partially or completely encrypted. These require the user to supply the encryption keys to NSTool so that it can process them. 
 

makefile

@@ -1,6 +1,6 @@
 # C++/C Recursive Project Makefile 
 # (c) Jack
-# Version 6 (20211110)
+# Version 8 (20220420)
 
 # Project Name
 PROJECT_NAME = nstool
@@ -23,13 +23,6 @@ ifeq ($(ROOT_PROJECT_NAME),)
 	export ROOT_PROJECT_DEPENDENCY_PATH = $(ROOT_PROJECT_PATH)/deps
 endif
 
-# Shared Library Definitions
-PROJECT_SO_VER_MAJOR = 0
-PROJECT_SO_VER_MINOR = 1
-PROJECT_SO_VER_PATCH = 0
-PROJECT_SONAME = $(PROJECT_NAME).so.$(PROJECT_SO_VER_MAJOR)
-PROJECT_SO_FILENAME = $(PROJECT_SONAME).$(PROJECT_SO_VER_MINOR).$(PROJECT_SO_VER_PATCH)
-
 # Project Dependencies
 PROJECT_DEPEND = pietendo toolchain fmt lz4 mbedtls
 PROJECT_DEPEND_LOCAL_DIR = libpietendo libtoolchain libfmt liblz4 libmbedtls
@@ -82,6 +75,7 @@ ifeq ($(PROJECT_PLATFORM), WIN32)
 	# Windows Flags/Libs
 	CC = x86_64-w64-mingw32-gcc
 	CXX = x86_64-w64-mingw32-g++
+	DEFINEFLAGS =
 	WARNFLAGS = -Wall -Wno-unused-value -Wno-unused-but-set-variable
 	ARCHFLAGS =
 	INC +=
@@ -91,6 +85,7 @@ else ifeq ($(PROJECT_PLATFORM), GNU)
 	# GNU/Linux Flags/Libs
 	#CC = 
 	#CXX =
+	DEFINEFLAGS =
 	WARNFLAGS = -Wall -Wno-unused-value -Wno-unused-but-set-variable
 	ARCHFLAGS =
 	INC +=
@@ -100,27 +95,27 @@ else ifeq ($(PROJECT_PLATFORM), MACOS)
 	# MacOS Flags/Libs
 	#CC = 
 	#CXX =
+	DEFINEFLAGS =
 	WARNFLAGS = -Wall -Wno-unused-value -Wno-unused-private-field
 	ARCHFLAGS = -arch $(PROJECT_PLATFORM_ARCH)
 	INC +=
 	LIB +=
-	ARFLAGS = rc	
+	ARFLAGS = rc
 endif
 
 # Compiler Flags
-CXXFLAGS = -std=c++11 $(INC) $(WARNFLAGS) $(ARCHFLAGS) -fPIC
-CFLAGS = -std=c11 $(INC) $(WARNFLAGS) $(ARCHFLAGS) -fPIC
+CXXFLAGS = -std=c++11 $(INC) $(DEFINEFLAGS) $(WARNFLAGS) $(ARCHFLAGS) -fPIC
+CFLAGS = -std=c11 $(INC) $(DEFINEFLAGS) $(WARNFLAGS) $(ARCHFLAGS) -fPIC
 
 # Object Files
 SRC_OBJ = $(foreach dir,$(PROJECT_SRC_SUBDIRS),$(subst .cpp,.o,$(wildcard $(dir)/*.cpp))) $(foreach dir,$(PROJECT_SRC_SUBDIRS),$(subst .cc,.o,$(wildcard $(dir)/*.cc))) $(foreach dir,$(PROJECT_SRC_SUBDIRS),$(subst .c,.o,$(wildcard $(dir)/*.c)))
 TESTSRC_OBJ = $(foreach dir,$(PROJECT_TESTSRC_SUBDIRS),$(subst .cpp,.o,$(wildcard $(dir)/*.cpp))) $(foreach dir,$(PROJECT_TESTSRC_SUBDIRS),$(subst .cc,.o,$(wildcard $(dir)/*.cc))) $(foreach dir,$(PROJECT_TESTSRC_SUBDIRS),$(subst .c,.o,$(wildcard $(dir)/*.c)))
 
 # all is the default, user should specify what the default should do
-#	- 'static_lib' for building static library
-#	- 'shared_lib' for building shared library
-#	- 'program' for building the program
+#	- 'static_lib' for building source as a static library
+#	- 'program' for building source as executable program
 #	- 'test_program' for building the test program
-# These can typically be used together however *_lib and program should not be used together
+# test_program can be used with program or static_lib, but program and static_lib cannot be used together
 all: program
 
 clean: clean_object_files remove_binary_dir
@@ -158,10 +153,6 @@ static_lib: $(SRC_OBJ) create_binary_dir
 	@echo LINK $(PROJECT_BIN_PATH)/$(PROJECT_NAME).a
 	@ar $(ARFLAGS) "$(PROJECT_BIN_PATH)/$(PROJECT_NAME).a" $(SRC_OBJ)
 
-shared_lib: $(SRC_OBJ) create_binary_dir
-	@echo LINK $(PROJECT_BIN_PATH)/$(PROJECT_SO_FILENAME)
-	@gcc -shared -Wl,-soname,$(PROJECT_SONAME) -o "$(PROJECT_BIN_PATH)/$(PROJECT_SO_FILENAME)" $(SRC_OBJ)
-
 # Build Program
 program: $(SRC_OBJ) create_binary_dir
 	@echo LINK $(PROJECT_BIN_PATH)/$(PROJECT_NAME)

src/EsCertProcess.cpp

@@ -104,14 +104,14 @@ void nstool::EsCertProcess::displayCert(const pie::hac::es::SignedData<pie::hac:
 
 	fmt::print("  SignType       {:s}", getSignTypeStr(cert.getSignature().getSignType()));
 	if (mCliOutputMode.show_extended_info)
-		fmt::print(" (0x{:x}) ({:s})", cert.getSignature().getSignType(), getEndiannessStr(cert.getSignature().isLittleEndian()));
+		fmt::print(" (0x{:x}) ({:s})", (uint32_t)cert.getSignature().getSignType(), getEndiannessStr(cert.getSignature().isLittleEndian()));
 	fmt::print("\n");
 
 	fmt::print("  Issuer:        {:s}\n", cert.getBody().getIssuer());
 	fmt::print("  Subject:       {:s}\n", cert.getBody().getSubject());
 	fmt::print("  PublicKeyType: {:s}", getPublicKeyTypeStr(cert.getBody().getPublicKeyType()));
 	if (mCliOutputMode.show_extended_info)
-		fmt::print(" ({:d})", cert.getBody().getPublicKeyType());
+		fmt::print(" ({:d})", (uint32_t)cert.getBody().getPublicKeyType());
 	fmt::print("\n");
 	fmt::print("  CertID:        0x{:x}\n", cert.getBody().getCertId());
 

src/EsTikProcess.cpp

@@ -85,8 +85,8 @@ void nstool::EsTikProcess::verifyTicket()
 		tc::crypto::GenerateSha1Hash(tik_hash.data(), mTik.getBody().getBytes().data(), mTik.getBody().getBytes().size());
 		break;
 	case (pie::hac::es::sign::HASH_ALGO_SHA256):
-		tik_hash = tc::ByteData(tc::crypto::Sha256Generator::kHashSize);
-		tc::crypto::GenerateSha256Hash(tik_hash.data(), mTik.getBody().getBytes().data(), mTik.getBody().getBytes().size());
+		tik_hash = tc::ByteData(tc::crypto::Sha2256Generator::kHashSize);
+		tc::crypto::GenerateSha2256Hash(tik_hash.data(), mTik.getBody().getBytes().data(), mTik.getBody().getBytes().size());
 		break;
 	}
 
@@ -109,7 +109,7 @@ void nstool::EsTikProcess::displayTicket()
 	fmt::print("[ES Ticket]\n");
 	fmt::print("  SignType:         {:s}", getSignTypeStr(mTik.getSignature().getSignType()));
 	if (mCliOutputMode.show_extended_info)
-		fmt::print(" (0x{:x})", mTik.getSignature().getSignType());
+		fmt::print(" (0x{:x})", (uint32_t)mTik.getSignature().getSignType());
 	fmt::print("\n");
 
 	fmt::print("  Issuer:           {:s}\n", body.getIssuer());

src/GameCardProcess.cpp

@@ -114,7 +114,7 @@ void nstool::GameCardProcess::importHeader()
 	pie::hac::sGcHeader_Rsa2048Signed* hdr_ptr = (pie::hac::sGcHeader_Rsa2048Signed*)(scratch.data() + mGcHeaderOffset);
 
 	// generate hash of raw header
-	tc::crypto::GenerateSha256Hash(mHdrHash.data(), (byte_t*)&hdr_ptr->header, sizeof(pie::hac::sGcHeader));
+	tc::crypto::GenerateSha2256Hash(mHdrHash.data(), (byte_t*)&hdr_ptr->header, sizeof(pie::hac::sGcHeader));
 
 	// save the signature
 	memcpy(mHdrSignature.data(), hdr_ptr->signature.data(), mHdrSignature.size());
@@ -225,7 +225,7 @@ bool nstool::GameCardProcess::validateRegionOfFile(int64_t offset, int64_t len,
 	mFile->read(scratch.data(), scratch.size());
 
 	// update hash
-	tc::crypto::Sha256Generator sha256_gen;
+	tc::crypto::Sha2256Generator sha256_gen;
 	sha256_gen.initialize();
 	sha256_gen.update(scratch.data(), scratch.size());
 	if (use_salt)
@@ -247,7 +247,7 @@ void nstool::GameCardProcess::validateXciSignature()
 {
 	if (mKeyCfg.xci_header_sign_key.isSet())
 	{
-		if (tc::crypto::VerifyRsa2048Pkcs1Sha256(mHdrSignature.data(), mHdrHash.data(), mKeyCfg.xci_header_sign_key.get()) == false)
+		if (tc::crypto::VerifyRsa2048Pkcs1Sha2256(mHdrSignature.data(), mHdrHash.data(), mKeyCfg.xci_header_sign_key.get()) == false)
 		{
 			fmt::print("[WARNING] GameCard Header Signature: FAIL\n");
 		}

src/NcaProcess.cpp

@@ -6,6 +6,7 @@
 #include <pietendo/hac/AesKeygen.h>
 #include <pietendo/hac/HierarchicalSha256Stream.h>
 #include <pietendo/hac/HierarchicalIntegrityStream.h>
+#include <pietendo/hac/BKTREncryptedStream.h>
 #include <pietendo/hac/PartitionFsSnapshotGenerator.h>
 #include <pietendo/hac/RomFsSnapshotGenerator.h>
 #include <pietendo/hac/CombinedFsSnapshotGenerator.h>
@@ -48,6 +49,11 @@ void nstool::NcaProcess::setInputFile(const std::shared_ptr<tc::io::IStream>& fi
 	mFile = file;
 }
 
+void nstool::NcaProcess::setBaseNcaPath(const tc::Optional<tc::io::Path>& nca_path)
+{
+	mBaseNcaPath = nca_path;
+}
+
 void nstool::NcaProcess::setKeyCfg(const KeyBag& keycfg)
 {
 	mKeyCfg = keycfg;
@@ -110,7 +116,7 @@ void nstool::NcaProcess::importHeader()
 	pie::hac::ContentArchiveUtil::decryptContentArchiveHeader((byte_t*)&mHdrBlock, (byte_t*)&mHdrBlock, mKeyCfg.nca_header_key.get());
 
 	// generate header hash
-	tc::crypto::GenerateSha256Hash(mHdrHash.data(), (byte_t*)&mHdrBlock.header, sizeof(pie::hac::sContentArchiveHeader));
+	tc::crypto::GenerateSha2256Hash(mHdrHash.data(), (byte_t*)&mHdrBlock.header, sizeof(pie::hac::sContentArchiveHeader));
 
 	// proccess main header
 	mHdr.fromBytes((byte_t*)&mHdrBlock.header, sizeof(pie::hac::sContentArchiveHeader));
@@ -211,6 +217,32 @@ void nstool::NcaProcess::generateNcaBodyEncryptionKeys()
 	}
 }
 
+nstool::NcaProcess nstool::NcaProcess::readBaseNCA()
+{
+	// open base nca stream
+	if (mBaseNcaPath.isNull())
+	{
+		throw tc::Exception(mModuleName, "Base NCA not supplied. Necessary for update NCA.");
+	}
+	std::shared_ptr<tc::io::IStream> base_stream = std::make_shared<tc::io::FileStream>(tc::io::FileStream(mBaseNcaPath.get(), tc::io::FileMode::Open, tc::io::FileAccess::Read));
+
+	// process base nca with output suppressed
+	NcaProcess obj;
+	nstool::CliOutputMode cliOutput;
+	cliOutput.show_basic_info = false;
+	cliOutput.show_extended_info = false;
+	cliOutput.show_keydata = false;
+	cliOutput.show_layout = false;
+	obj.setCliOutputMode(cliOutput);
+	obj.setVerifyMode(true);
+	obj.setKeyCfg(mKeyCfg);
+	obj.setInputFile(base_stream);
+	obj.process();
+
+	// return processed base nca
+	return obj;
+}
+
 void nstool::NcaProcess::generatePartitionConfiguration()
 {
 	for (size_t i = 0; i < mHdr.getPartitionEntryList().size(); i++)
@@ -224,7 +256,7 @@ void nstool::NcaProcess::generatePartitionConfiguration()
 
 		// validate header hash
 		pie::hac::detail::sha256_hash_t fs_header_hash;
-		tc::crypto::GenerateSha256Hash(fs_header_hash.data(), (const byte_t*)&mHdrBlock.fs_header[partition.header_index], sizeof(pie::hac::sContentArchiveFsHeader));
+		tc::crypto::GenerateSha2256Hash(fs_header_hash.data(), (const byte_t*)&mHdrBlock.fs_header[partition.header_index], sizeof(pie::hac::sContentArchiveFsHeader));
 		if (fs_header_hash != partition.fs_header_hash)
 		{
 			throw tc::Exception(mModuleName, fmt::format("NCA FS Header [{:d}] Hash: FAIL", partition.header_index));
@@ -266,13 +298,13 @@ void nstool::NcaProcess::generatePartitionConfiguration()
 			else
 			{
 				// create raw partition
-				info.reader = std::make_shared<tc::io::SubStream>(tc::io::SubStream(mFile, info.offset, info.size));
+				info.raw_reader = std::make_shared<tc::io::SubStream>(tc::io::SubStream(mFile, info.offset, info.size));
 
 				// handle encryption if required reader based on encryption type
 				if (info.enc_type == pie::hac::nca::EncryptionType_None)
 				{
 					// no encryption so do nothing
-					//info.reader = info.reader;
+					info.decrypt_reader = info.raw_reader;
 				}
 				else if (info.enc_type == pie::hac::nca::EncryptionType_AesCtr)
 				{
@@ -284,13 +316,49 @@ void nstool::NcaProcess::generatePartitionConfiguration()
 
 					// get partition counter
 					pie::hac::detail::aes_iv_t partition_ctr = info.aes_ctr;
-					tc::crypto::IncrementCounterAes128Ctr(partition_ctr.data(), info.offset>>4);
+					tc::crypto::IncrementCounterAes128Ctr(partition_ctr.data(), info.offset >> 4);
 
 					// create decryption stream
-					info.reader = std::make_shared<tc::crypto::Aes128CtrEncryptedStream>(tc::crypto::Aes128CtrEncryptedStream(info.reader, partition_key, partition_ctr));
+					info.decrypt_reader = std::make_shared<tc::crypto::Aes128CtrEncryptedStream>(tc::crypto::Aes128CtrEncryptedStream(info.raw_reader, partition_key, partition_ctr));
+				}
+				else if (info.enc_type == pie::hac::nca::EncryptionType_AesCtrEx)
+				{
+					if (mContentKey.aes_ctr.isNull())
+						throw tc::Exception(mModuleName, "AES-CTR Key was not determined");
+
+					// get partition key
+					pie::hac::detail::aes128_key_t partition_key = mContentKey.aes_ctr.get();
+
+					// get partition counter
+					pie::hac::detail::aes_iv_t partition_ctr = info.aes_ctr;
+					tc::crypto::IncrementCounterAes128Ctr(partition_ctr.data(), info.offset >> 4);
+
+					// TODO see if AesCtrEx encryption can just be for creating the transparent decryption, with IndirectStorage IStream construction being done after decryption but before hash layer processing
+					// this might be relevant when processing compressed or sparse storage
+
+					NcaProcess nca_base = readBaseNCA();
+					if (nca_base.mHdr.getProgramId() != mHdr.getProgramId())
+					{
+						throw tc::Exception(mModuleName, "Invalid base nca. ProgramID diferent.");
+					}
 
+					std::shared_ptr<tc::io::IStream> base_reader;
+					for (auto& partition_base : nca_base.mPartitions)
+					{
+						if (partition_base.format_type == pie::hac::nca::FormatType::FormatType_RomFs && partition_base.raw_reader != nullptr)
+						{
+							base_reader = partition_base.decrypt_reader;
+						}
+					}
+					if (base_reader == nullptr)
+					{
+						throw tc::Exception(mModuleName, "Cannot determine RomFs from base nca.");
+					}
+
+					// create decryption stream
+					info.decrypt_reader = std::make_shared<pie::hac::BKTREncryptedStream>(pie::hac::BKTREncryptedStream(info.raw_reader, partition_key, partition_ctr, fs_header.patch_info, base_reader));
 				}
-				else if (info.enc_type == pie::hac::nca::EncryptionType_AesXts || info.enc_type == pie::hac::nca::EncryptionType_AesCtrEx)
+				else if (info.enc_type == pie::hac::nca::EncryptionType_AesXts)
 				{
 					throw tc::Exception(mModuleName, fmt::format("EncryptionType({:s}): UNSUPPORTED", pie::hac::ContentArchiveUtil::getEncryptionTypeAsString(info.enc_type)));
 				}
@@ -304,12 +372,14 @@ void nstool::NcaProcess::generatePartitionConfiguration()
 			switch (info.hash_type)
 			{
 			case (pie::hac::nca::HashType_None):
+				// no hash layer, do nothing
+				info.reader = info.decrypt_reader;
 				break;
 			case (pie::hac::nca::HashType_HierarchicalSha256):
-				info.reader = std::make_shared<pie::hac::HierarchicalSha256Stream>(pie::hac::HierarchicalSha256Stream(info.reader, info.hierarchicalsha256_hdr));
+				info.reader = std::make_shared<pie::hac::HierarchicalSha256Stream>(pie::hac::HierarchicalSha256Stream(info.decrypt_reader, info.hierarchicalsha256_hdr));
 				break;
 			case (pie::hac::nca::HashType_HierarchicalIntegrity):
-				info.reader = std::make_shared<pie::hac::HierarchicalIntegrityStream>(pie::hac::HierarchicalIntegrityStream(info.reader, info.hierarchicalintegrity_hdr));
+				info.reader = std::make_shared<pie::hac::HierarchicalIntegrityStream>(pie::hac::HierarchicalIntegrityStream(info.decrypt_reader, info.hierarchicalintegrity_hdr));
 				break;
 			default:
 				throw tc::Exception(mModuleName, fmt::format("HashType({:s}): UNKNOWN", pie::hac::ContentArchiveUtil::getHashTypeAsString(info.hash_type)));
@@ -342,7 +412,7 @@ void nstool::NcaProcess::validateNcaSignatures()
 	// validate signature[0]
 	if (mKeyCfg.nca_header_sign0_key.find(mHdr.getSignatureKeyGeneration()) != mKeyCfg.nca_header_sign0_key.end())
 	{
-		if (tc::crypto::VerifyRsa2048PssSha256(mHdrBlock.signature_main.data(), mHdrHash.data(), mKeyCfg.nca_header_sign0_key[mHdr.getSignatureKeyGeneration()]) == false)
+		if (tc::crypto::VerifyRsa2048PssSha2256(mHdrBlock.signature_main.data(), mHdrHash.data(), mKeyCfg.nca_header_sign0_key[mHdr.getSignatureKeyGeneration()]) == false)
 		{
 			fmt::print("[WARNING] NCA Header Main Signature: FAIL\n");
 		}
@@ -376,7 +446,7 @@ void nstool::NcaProcess::validateNcaSignatures()
 					npdm.setCliOutputMode(CliOutputMode(false, false, false, false));
 					npdm.process();
 
-					if (tc::crypto::VerifyRsa2048PssSha256(mHdrBlock.signature_acid.data(), mHdrHash.data(), npdm.getMeta().getAccessControlInfoDesc().getContentArchiveHeaderSignature2Key()) == false)
+					if (tc::crypto::VerifyRsa2048PssSha2256(mHdrBlock.signature_acid.data(), mHdrHash.data(), npdm.getMeta().getAccessControlInfoDesc().getContentArchiveHeaderSignature2Key()) == false)
 					{
 						throw tc::Exception("Bad signature");
 					}

src/NcaProcess.h

@@ -21,6 +21,8 @@ class NcaProcess
 	void setKeyCfg(const KeyBag& keycfg);
 	void setCliOutputMode(CliOutputMode type);
 	void setVerifyMode(bool verify);
+	void setBaseNcaPath(const tc::Optional<tc::io::Path>& nca_path);
+
 
 	// fs specific
 	void setShowFsTree(bool show_fs_tree);
@@ -39,6 +41,7 @@ class NcaProcess
 	KeyBag mKeyCfg;
 	CliOutputMode mCliOutputMode;
 	bool mVerify;
+	tc::Optional<tc::io::Path> mBaseNcaPath;
 
 	// fs processing
 	std::shared_ptr<tc::io::IFileSystem> mFileSystem;
@@ -93,7 +96,9 @@ class NcaProcess
 	// raw partition data
 	struct sPartitionInfo
 	{
-		std::shared_ptr<tc::io::IStream> reader;
+		std::shared_ptr<tc::io::IStream> raw_reader; // raw unprocessed partition stream
+		std::shared_ptr<tc::io::IStream> decrypt_reader; // partition stream with transparent decryption
+		std::shared_ptr<tc::io::IStream> reader; // partition stream with transparent decryption & hash layer processing
 		tc::io::VirtualFileSystem::FileSystemSnapshot fs_snapshot;
 		std::shared_ptr<tc::io::IFileSystem> fs_reader;
 		std::string fail_reason;
@@ -126,6 +131,8 @@ class NcaProcess
 	void displayHeader();
 	void processPartitions();
 
+	NcaProcess readBaseNCA();
+
 	std::string getContentTypeForMountStr(pie::hac::nca::ContentType cont_type) const;
 };