fix: I/O failures causing a crash + unify logging and assertions

ue4cli/CachedDataManager.py

@@ -1,6 +1,7 @@
 from .ConfigurationManager import ConfigurationManager
 from .JsonDataManager import JsonDataManager
-import os, shutil
+from .Utility import Utility
+import os
 
 class CachedDataManager(object):
 	"""
@@ -13,7 +14,7 @@ def clearCache():
 		Clears any cached data we have stored about specific engine versions
 		"""
 		if os.path.exists(CachedDataManager._cacheDir()) == True:
-			shutil.rmtree(CachedDataManager._cacheDir())
+			Utility.removeDir(CachedDataManager._cacheDir())
 
 	@staticmethod
 	def getCachedDataKey(engineVersionHash, key):

ue4cli/JsonDataManager.py

@@ -1,6 +1,7 @@
 from .UnrealManagerException import UnrealManagerException
 from .Utility import Utility
-import json, os, platform
+from .UtilityException import UtilityException
+import json, os
 
 class JsonDataManager(object):
 	"""
@@ -12,6 +13,19 @@ def __init__(self, jsonFile):
 		Creates a new JsonDataManager instance for the specified JSON file
 		"""
 		self.jsonFile = jsonFile
+
+	def loads(self):
+		"""
+		Reads and loads owned jsonFile
+		"""
+		try:
+			path = self.jsonFile
+			file = Utility.readFile(path)
+			return json.loads(file)
+		except json.JSONDecodeError as e:
+			# FIXME: This is the only place outside of Utility class where we use UtilityException.
+			# Not worth to create new Exception class for only one single case, at least not now.
+			raise UtilityException(f'failed to load "{str(path)}" due to: ({type(e).__name__}) {str(e)}') from e
 
 	def getKey(self, key):
 		"""
@@ -28,10 +42,7 @@ def getDictionary(self):
 		Retrieves the entire data dictionary
 		"""
 		if os.path.exists(self.jsonFile):
-			try:
-				return json.loads(Utility.readFile(self.jsonFile))
-			except json.JSONDecodeError as err:
-				raise UnrealManagerException('malformed JSON configuration file "{}" ({})'.format(self.jsonFile, err))
+			return self.loads()
 		else:
 			return {}
 
@@ -51,7 +62,7 @@ def setDictionary(self, data):
 		# Create the directory containing the JSON file if it doesn't already exist
 		jsonDir = os.path.dirname(self.jsonFile)
 		if os.path.exists(jsonDir) == False:
-			os.makedirs(jsonDir)
+			Utility.makeDirs(jsonDir)
 
 		# Store the dictionary
 		Utility.writeFile(self.jsonFile, json.dumps(data))

ue4cli/UE4BuildInterrogator.py

@@ -2,11 +2,15 @@
 from .UnrealManagerException import UnrealManagerException
 from .CachedDataManager import CachedDataManager
 from .Utility import Utility
-import json, os, platform, shutil, tempfile
+from .UtilityException import UtilityException
+from .JsonDataManager import JsonDataManager
+import os, tempfile
 
 class UE4BuildInterrogator(object):
 
 	def __init__(self, engineRoot, engineVersion, engineVersionHash, runUBTFunc):
+		# WARN: os.path.realpath can potentially fail with OSError,
+		# but if it ever happens, this is most likely bug in our code
 		self.engineRoot = os.path.realpath(engineRoot)
 		self.engineSourceDir = 'Engine/Source/'
 		self.engineVersion = engineVersion
@@ -160,7 +164,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 		sentinelBackup = sentinelFile + '.bak'
 		renameSentinel = os.path.exists(sentinelFile) and os.environ.get('UE4CLI_SENTINEL_RENAME', '0') == '1'
 		if renameSentinel == True:
-			shutil.move(sentinelFile, sentinelBackup)
+			Utility.moveFile(sentinelFile, sentinelBackup)
 
 		# Invoke UnrealBuildTool in JSON export mode (make sure we specify gathering mode, since this is a prerequisite of JSON export)
 		# (Ensure we always perform sentinel file cleanup even when errors occur)
@@ -172,10 +176,10 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 				self.runUBTFunc('UE4Editor', platformIdentifier, configuration, args)
 		finally:
 			if renameSentinel == True:
-				shutil.move(sentinelBackup, sentinelFile)
+				Utility.moveFile(sentinelBackup, sentinelFile)
 
 		# Parse the JSON output
-		result = json.loads(Utility.readFile(jsonFile))
+		result = JsonDataManager(jsonFile).loads()
 
 		# Extract the list of third-party library modules
 		# (Note that since UE4.21, modules no longer have a "Type" field, so we must
@@ -188,7 +192,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 
 		# Remove the temp directory
 		try:
-			shutil.rmtree(tempDir)
+			Utility.removeDir(tempDir)
 		except:
 			pass
 

ue4cli/UnrealManagerBase.py

@@ -5,7 +5,9 @@
 from .CachedDataManager import CachedDataManager
 from .CMakeCustomFlags import CMakeCustomFlags
 from .Utility import Utility
-import glob, hashlib, json, os, re, shutil, sys
+from .JsonDataManager import JsonDataManager
+from .UtilityException import UtilityException
+import glob, hashlib, json, os, re
 
 class UnrealManagerBase(object):
 	"""
@@ -44,13 +46,13 @@ def setEngineRootOverride(self, rootDir):
 		# Set the new root directory
 		rootDir = os.path.abspath(rootDir)
 		ConfigurationManager.setConfigKey('rootDirOverride', rootDir)
-		print('Set engine root path override: {}'.format(rootDir))
+		Utility.printStderr('Setting engine root path override:', str(rootDir))
 
 		# Check that the specified directory is valid and warn the user if it is not
 		try:
 			self.getEngineVersion()
 		except:
-			print('Warning: the specified directory does not appear to contain a valid version of the Unreal Engine.')
+			raise UnrealManagerException('the specified directory does not appear to contain a valid version of the Unreal Engine.') from None
 
 	def clearEngineRootOverride(self):
 		"""
@@ -94,7 +96,7 @@ def getEngineVersion(self, outputFormat = 'full'):
 
 		# Verify that the requested output format is valid
 		if outputFormat not in formats:
-			raise Exception('unreconised version output format "{}"'.format(outputFormat))
+			raise UnrealManagerException(f'unreconised version output format "{str(outputFormat)}"')
 
 		return formats[outputFormat]
 
@@ -174,7 +176,7 @@ def getDescriptor(self, dir):
 			try:
 				return self.getPluginDescriptor(dir)
 			except:
-				raise UnrealManagerException('could not detect an Unreal project or plugin in the directory "{}"'.format(dir))
+				raise UnrealManagerException(f'could not detect an Unreal project or plugin in the directory "{str(dir)}"') from None
 
 	def isProject(self, descriptor):
 		"""
@@ -316,8 +318,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]):
 
 		# If the project is a pure Blueprint project, then we cannot generate project files
 		if os.path.exists(os.path.join(dir, 'Source')) == False:
-			Utility.printStderr('Pure Blueprint project, nothing to generate project files for.')
-			return
+			raise UnrealManagerException('Pure Blueprint project, nothing to generate project files for.')
 
 		# Generate the project files
 		genScript = self.getGenerateScript()
@@ -334,8 +335,8 @@ def cleanDescriptor(self, dir=os.getcwd()):
 
 		# Because performing a clean will also delete the engine build itself when using
 		# a source build, we simply delete the `Binaries` and `Intermediate` directories
-		shutil.rmtree(os.path.join(dir, 'Binaries'), ignore_errors=True)
-		shutil.rmtree(os.path.join(dir, 'Intermediate'), ignore_errors=True)
+		Utility.removeDir(os.path.join(dir, 'Binaries'), ignore_errors=True)
+		Utility.removeDir(os.path.join(dir, 'Intermediate'), ignore_errors=True)
 
 		# If we are cleaning a project, also clean any plugins
 		if self.isProject(descriptor):
@@ -354,12 +355,11 @@ def buildDescriptor(self, dir=os.getcwd(), configuration='Development', target='
 
 		# If the project or plugin is Blueprint-only, there is no C++ code to build
 		if os.path.exists(os.path.join(dir, 'Source')) == False:
-			Utility.printStderr('Pure Blueprint {}, nothing to build.'.format(descriptorType))
-			return
+			raise UnrealManagerException(f'Pure Blueprint {str(descriptorType)}, nothing to build.')
 
 		# Verify that the specified build configuration is valid
 		if configuration not in self.validBuildConfigurations():
-			raise UnrealManagerException('invalid build configuration "' + configuration + '"')
+			raise UnrealManagerException(f'invalid build configuration "{str(configuration)}"')
 
 		# Check if the user specified the `-notools` flag to opt out of building Engine tools when working with source builds
 		unstripped = list(args)
@@ -408,7 +408,7 @@ def packageProject(self, dir=os.getcwd(), configuration='Shipping', extraArgs=[]
 
 		# Verify that the specified build configuration is valid
 		if configuration not in self.validBuildConfigurations():
-			raise UnrealManagerException('invalid build configuration "' + configuration + '"')
+			raise UnrealManagerException(f'invalid build configuration "{str(configuration)}"')
 
 		# Strip out the `-NoCompileEditor` flag if the user has specified it, since the Development version
 		# of the Editor modules for the project are needed in order to run the commandlet that cooks content
@@ -538,10 +538,12 @@ def listAutomationTests(self, projectFile):
 		# Detect if the Editor terminated abnormally (i.e. not triggered by `automation quit`)
 		# In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus
 		if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout:
-			raise RuntimeError(
-				'failed to retrieve the list of automation tests!' +
-				' stdout was: "{}", stderr was: "{}"'.format(logOutput.stdout, logOutput.stderr)
-			)
+			Utility.printStderr("Warning: abnormal Editor termination detected!")
+			Utility.printStderr("printing stdout..")
+			print(logOutput.stdout)
+			Utility.printStderr("printing stderr..")
+			print(logOutput.stderr)
+			raise UnrealManagerException('failed to retrieve the list of automation tests!')
 
 		return sorted(list(tests))
 
@@ -556,7 +558,7 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 
 		# Verify that at least one argument was supplied
 		if len(args) == 0:
-			raise RuntimeError('at least one test name must be specified')
+			raise UnrealManagerException('at least one test name must be specified')
 
 		# Gather any additional arguments to pass directly to the Editor
 		extraArgs = []
@@ -567,7 +569,12 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 
 		# Build the project if it isn't already built
 		Utility.printStderr('Ensuring project is built...')
-		self.buildDescriptor(dir, suppressOutput=True)
+		try:
+			self.buildDescriptor(dir, suppressOutput=True)
+		except UnrealManagerException:
+			# FIXME: Ideally, we should NOT catch every UnrealManagerException here
+			# This is currently a limitation of our API that uses only one Exception class for multiple different cases
+			pass
 
 		# Determine which arguments we are passing to the automation test commandlet
 		projectFile = self.getProjectDescriptor(dir)
@@ -605,7 +612,7 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 			# Detect abnormal exit conditions (those not triggered by `automation quit`)
 			# In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus
 			if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout:
-				sys.exit(1)
+				raise UnrealManagerException('abnormal exit condition detected')
 
 			# Since UE4 doesn't consistently produce accurate exit codes across all platforms, we need to rely on
 			# text-based heuristics to detect failed automation tests or errors related to not finding any tests to run
@@ -618,12 +625,12 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 			]
 			for errorStr in errorStrings:
 				if errorStr in logOutput.stdout:
-					sys.exit(1)
+					raise UnrealManagerException('abnormal exit condition detected')
 
 			# If an explicit exit code was specified in the output text then identify it and propagate it
-			match = re.search('TEST COMPLETE\\. EXIT CODE: ([0-9]+)', logOutput.stdout + logOutput.stderr)
+			match = re.search('TEST COMPLETE\\. EXIT CODE: ([1-9]+)', logOutput.stdout + logOutput.stderr)
 			if match is not None:
-				sys.exit(int(match.group(1)))
+				raise UnrealManagerException('abnormal exit condition detected')
 
 	# "Protected" methods
 
@@ -638,7 +645,7 @@ def _getEngineVersionDetails(self):
 		Parses the JSON version details for the latest installed version of UE4
 		"""
 		versionFile = os.path.join(self.getEngineRoot(), 'Engine', 'Build', 'Build.version')
-		return json.loads(Utility.readFile(versionFile))
+		return JsonDataManager(versionFile).loads()
 
 	def _getEngineVersionHash(self):
 		"""

ue4cli/UnrealManagerWindows.py

@@ -42,7 +42,7 @@ def getGenerateScript(self):
 		except:
 			pass
 
-		raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe.\nThis typically indicates that .uproject files are not correctly associated with UE4.')
+		raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe. This typically indicates that .uproject files are not correctly associated with UE4.')
 
 	def getRunUATScript(self):
 		return self.getEngineRoot() + '\\Engine\\Build\\BatchFiles\\RunUAT.bat'
@@ -52,7 +52,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]):
 		# If we are using our custom batch file, use the appropriate arguments
 		genScript = self.getGenerateScript()
 		projectFile = self.getProjectDescriptor(dir)
-		print(projectFile)
+		Utility.printStderr('Using project file:', projectFile)
 		if '.ue4\\GenerateProjectFiles.bat' in genScript:
 			Utility.run([genScript, projectFile], raiseOnError=True)
 		else:
@@ -91,7 +91,7 @@ def _customBatchScriptDir(self):
 		# If the script directory doesn't already exist, attempt to create it
 		scriptDir = os.path.join(os.environ['HOMEDRIVE'] + os.environ['HOMEPATH'], '.ue4')
 		try:
-			os.makedirs(scriptDir)
+			Utility.makeDirs(scriptDir)
 		except:
 			pass