diff --git a/ue4cli/CachedDataManager.py b/ue4cli/CachedDataManager.py index 9b03753..34678a7 100644 --- a/ue4cli/CachedDataManager.py +++ b/ue4cli/CachedDataManager.py @@ -1,6 +1,7 @@ from .ConfigurationManager import ConfigurationManager from .JsonDataManager import JsonDataManager -import os, shutil +from .Utility import Utility +import os class CachedDataManager(object): """ @@ -13,7 +14,7 @@ def clearCache(): Clears any cached data we have stored about specific engine versions """ if os.path.exists(CachedDataManager._cacheDir()) == True: - shutil.rmtree(CachedDataManager._cacheDir()) + Utility.removeDir(CachedDataManager._cacheDir()) @staticmethod def getCachedDataKey(engineVersionHash, key): diff --git a/ue4cli/JsonDataManager.py b/ue4cli/JsonDataManager.py index 53fbb77..049f762 100644 --- a/ue4cli/JsonDataManager.py +++ b/ue4cli/JsonDataManager.py @@ -1,6 +1,7 @@ from .UnrealManagerException import UnrealManagerException from .Utility import Utility -import json, os, platform +from .UtilityException import UtilityException +import json, os class JsonDataManager(object): """ @@ -12,6 +13,19 @@ def __init__(self, jsonFile): Creates a new JsonDataManager instance for the specified JSON file """ self.jsonFile = jsonFile + + def loads(self): + """ + Reads and loads owned jsonFile + """ + try: + path = self.jsonFile + file = Utility.readFile(path) + return json.loads(file) + except json.JSONDecodeError as e: + # FIXME: This is the only place outside of Utility class where we use UtilityException. + # Not worth to create new Exception class for only one single case, at least not now. + raise UtilityException(f'failed to load "{str(path)}" due to: ({type(e).__name__}) {str(e)}') from e def getKey(self, key): """ @@ -28,10 +42,7 @@ def getDictionary(self): Retrieves the entire data dictionary """ if os.path.exists(self.jsonFile): - try: - return json.loads(Utility.readFile(self.jsonFile)) - except json.JSONDecodeError as err: - raise UnrealManagerException('malformed JSON configuration file "{}" ({})'.format(self.jsonFile, err)) + return self.loads() else: return {} @@ -51,7 +62,7 @@ def setDictionary(self, data): # Create the directory containing the JSON file if it doesn't already exist jsonDir = os.path.dirname(self.jsonFile) if os.path.exists(jsonDir) == False: - os.makedirs(jsonDir) + Utility.makeDirs(jsonDir) # Store the dictionary Utility.writeFile(self.jsonFile, json.dumps(data)) diff --git a/ue4cli/UE4BuildInterrogator.py b/ue4cli/UE4BuildInterrogator.py index 12a42d8..098f056 100644 --- a/ue4cli/UE4BuildInterrogator.py +++ b/ue4cli/UE4BuildInterrogator.py @@ -2,11 +2,15 @@ from .UnrealManagerException import UnrealManagerException from .CachedDataManager import CachedDataManager from .Utility import Utility -import json, os, platform, shutil, tempfile +from .UtilityException import UtilityException +from .JsonDataManager import JsonDataManager +import os, tempfile class UE4BuildInterrogator(object): def __init__(self, engineRoot, engineVersion, engineVersionHash, runUBTFunc): + # WARN: os.path.realpath can potentially fail with OSError, + # but if it ever happens, this is most likely bug in our code self.engineRoot = os.path.realpath(engineRoot) self.engineSourceDir = 'Engine/Source/' self.engineVersion = engineVersion @@ -160,7 +164,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration): sentinelBackup = sentinelFile + '.bak' renameSentinel = os.path.exists(sentinelFile) and os.environ.get('UE4CLI_SENTINEL_RENAME', '0') == '1' if renameSentinel == True: - shutil.move(sentinelFile, sentinelBackup) + Utility.moveFile(sentinelFile, sentinelBackup) # Invoke UnrealBuildTool in JSON export mode (make sure we specify gathering mode, since this is a prerequisite of JSON export) # (Ensure we always perform sentinel file cleanup even when errors occur) @@ -172,10 +176,10 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration): self.runUBTFunc('UE4Editor', platformIdentifier, configuration, args) finally: if renameSentinel == True: - shutil.move(sentinelBackup, sentinelFile) + Utility.moveFile(sentinelBackup, sentinelFile) # Parse the JSON output - result = json.loads(Utility.readFile(jsonFile)) + result = JsonDataManager(jsonFile).loads() # Extract the list of third-party library modules # (Note that since UE4.21, modules no longer have a "Type" field, so we must @@ -188,7 +192,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration): # Remove the temp directory try: - shutil.rmtree(tempDir) + Utility.removeDir(tempDir) except: pass diff --git a/ue4cli/UnrealManagerBase.py b/ue4cli/UnrealManagerBase.py index 763557f..ff60e6a 100644 --- a/ue4cli/UnrealManagerBase.py +++ b/ue4cli/UnrealManagerBase.py @@ -5,7 +5,9 @@ from .CachedDataManager import CachedDataManager from .CMakeCustomFlags import CMakeCustomFlags from .Utility import Utility -import glob, hashlib, json, os, re, shutil, sys +from .JsonDataManager import JsonDataManager +from .UtilityException import UtilityException +import glob, hashlib, json, os, re class UnrealManagerBase(object): """ @@ -44,13 +46,13 @@ def setEngineRootOverride(self, rootDir): # Set the new root directory rootDir = os.path.abspath(rootDir) ConfigurationManager.setConfigKey('rootDirOverride', rootDir) - print('Set engine root path override: {}'.format(rootDir)) + Utility.printStderr('Setting engine root path override:', str(rootDir)) # Check that the specified directory is valid and warn the user if it is not try: self.getEngineVersion() except: - print('Warning: the specified directory does not appear to contain a valid version of the Unreal Engine.') + raise UnrealManagerException('the specified directory does not appear to contain a valid version of the Unreal Engine.') from None def clearEngineRootOverride(self): """ @@ -94,7 +96,7 @@ def getEngineVersion(self, outputFormat = 'full'): # Verify that the requested output format is valid if outputFormat not in formats: - raise Exception('unreconised version output format "{}"'.format(outputFormat)) + raise UnrealManagerException(f'unreconised version output format "{str(outputFormat)}"') return formats[outputFormat] @@ -174,7 +176,7 @@ def getDescriptor(self, dir): try: return self.getPluginDescriptor(dir) except: - raise UnrealManagerException('could not detect an Unreal project or plugin in the directory "{}"'.format(dir)) + raise UnrealManagerException(f'could not detect an Unreal project or plugin in the directory "{str(dir)}"') from None def isProject(self, descriptor): """ @@ -316,8 +318,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]): # If the project is a pure Blueprint project, then we cannot generate project files if os.path.exists(os.path.join(dir, 'Source')) == False: - Utility.printStderr('Pure Blueprint project, nothing to generate project files for.') - return + raise UnrealManagerException('Pure Blueprint project, nothing to generate project files for.') # Generate the project files genScript = self.getGenerateScript() @@ -334,8 +335,8 @@ def cleanDescriptor(self, dir=os.getcwd()): # Because performing a clean will also delete the engine build itself when using # a source build, we simply delete the `Binaries` and `Intermediate` directories - shutil.rmtree(os.path.join(dir, 'Binaries'), ignore_errors=True) - shutil.rmtree(os.path.join(dir, 'Intermediate'), ignore_errors=True) + Utility.removeDir(os.path.join(dir, 'Binaries'), ignore_errors=True) + Utility.removeDir(os.path.join(dir, 'Intermediate'), ignore_errors=True) # If we are cleaning a project, also clean any plugins if self.isProject(descriptor): @@ -354,12 +355,11 @@ def buildDescriptor(self, dir=os.getcwd(), configuration='Development', target=' # If the project or plugin is Blueprint-only, there is no C++ code to build if os.path.exists(os.path.join(dir, 'Source')) == False: - Utility.printStderr('Pure Blueprint {}, nothing to build.'.format(descriptorType)) - return + raise UnrealManagerException(f'Pure Blueprint {str(descriptorType)}, nothing to build.') # Verify that the specified build configuration is valid if configuration not in self.validBuildConfigurations(): - raise UnrealManagerException('invalid build configuration "' + configuration + '"') + raise UnrealManagerException(f'invalid build configuration "{str(configuration)}"') # Check if the user specified the `-notools` flag to opt out of building Engine tools when working with source builds unstripped = list(args) @@ -408,7 +408,7 @@ def packageProject(self, dir=os.getcwd(), configuration='Shipping', extraArgs=[] # Verify that the specified build configuration is valid if configuration not in self.validBuildConfigurations(): - raise UnrealManagerException('invalid build configuration "' + configuration + '"') + raise UnrealManagerException(f'invalid build configuration "{str(configuration)}"') # Strip out the `-NoCompileEditor` flag if the user has specified it, since the Development version # of the Editor modules for the project are needed in order to run the commandlet that cooks content @@ -538,10 +538,12 @@ def listAutomationTests(self, projectFile): # Detect if the Editor terminated abnormally (i.e. not triggered by `automation quit`) # In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout: - raise RuntimeError( - 'failed to retrieve the list of automation tests!' + - ' stdout was: "{}", stderr was: "{}"'.format(logOutput.stdout, logOutput.stderr) - ) + Utility.printStderr("Warning: abnormal Editor termination detected!") + Utility.printStderr("printing stdout..") + print(logOutput.stdout) + Utility.printStderr("printing stderr..") + print(logOutput.stderr) + raise UnrealManagerException('failed to retrieve the list of automation tests!') return sorted(list(tests)) @@ -556,7 +558,7 @@ def automationTests(self, dir=os.getcwd(), args=[]): # Verify that at least one argument was supplied if len(args) == 0: - raise RuntimeError('at least one test name must be specified') + raise UnrealManagerException('at least one test name must be specified') # Gather any additional arguments to pass directly to the Editor extraArgs = [] @@ -567,7 +569,12 @@ def automationTests(self, dir=os.getcwd(), args=[]): # Build the project if it isn't already built Utility.printStderr('Ensuring project is built...') - self.buildDescriptor(dir, suppressOutput=True) + try: + self.buildDescriptor(dir, suppressOutput=True) + except UnrealManagerException: + # FIXME: Ideally, we should NOT catch every UnrealManagerException here + # This is currently a limitation of our API that uses only one Exception class for multiple different cases + pass # Determine which arguments we are passing to the automation test commandlet projectFile = self.getProjectDescriptor(dir) @@ -605,7 +612,7 @@ def automationTests(self, dir=os.getcwd(), args=[]): # Detect abnormal exit conditions (those not triggered by `automation quit`) # In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout: - sys.exit(1) + raise UnrealManagerException('abnormal exit condition detected') # Since UE4 doesn't consistently produce accurate exit codes across all platforms, we need to rely on # text-based heuristics to detect failed automation tests or errors related to not finding any tests to run @@ -618,12 +625,12 @@ def automationTests(self, dir=os.getcwd(), args=[]): ] for errorStr in errorStrings: if errorStr in logOutput.stdout: - sys.exit(1) + raise UnrealManagerException('abnormal exit condition detected') # If an explicit exit code was specified in the output text then identify it and propagate it - match = re.search('TEST COMPLETE\\. EXIT CODE: ([0-9]+)', logOutput.stdout + logOutput.stderr) + match = re.search('TEST COMPLETE\\. EXIT CODE: ([1-9]+)', logOutput.stdout + logOutput.stderr) if match is not None: - sys.exit(int(match.group(1))) + raise UnrealManagerException('abnormal exit condition detected') # "Protected" methods @@ -638,7 +645,7 @@ def _getEngineVersionDetails(self): Parses the JSON version details for the latest installed version of UE4 """ versionFile = os.path.join(self.getEngineRoot(), 'Engine', 'Build', 'Build.version') - return json.loads(Utility.readFile(versionFile)) + return JsonDataManager(versionFile).loads() def _getEngineVersionHash(self): """ diff --git a/ue4cli/UnrealManagerWindows.py b/ue4cli/UnrealManagerWindows.py index 535c69b..e7a2101 100644 --- a/ue4cli/UnrealManagerWindows.py +++ b/ue4cli/UnrealManagerWindows.py @@ -42,7 +42,7 @@ def getGenerateScript(self): except: pass - raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe.\nThis typically indicates that .uproject files are not correctly associated with UE4.') + raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe. This typically indicates that .uproject files are not correctly associated with UE4.') def getRunUATScript(self): return self.getEngineRoot() + '\\Engine\\Build\\BatchFiles\\RunUAT.bat' @@ -52,7 +52,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]): # If we are using our custom batch file, use the appropriate arguments genScript = self.getGenerateScript() projectFile = self.getProjectDescriptor(dir) - print(projectFile) + Utility.printStderr('Using project file:', projectFile) if '.ue4\\GenerateProjectFiles.bat' in genScript: Utility.run([genScript, projectFile], raiseOnError=True) else: @@ -91,7 +91,7 @@ def _customBatchScriptDir(self): # If the script directory doesn't already exist, attempt to create it scriptDir = os.path.join(os.environ['HOMEDRIVE'] + os.environ['HOMEPATH'], '.ue4') try: - os.makedirs(scriptDir) + Utility.makeDirs(scriptDir) except: pass diff --git a/ue4cli/Utility.py b/ue4cli/Utility.py index 65e4e12..d8989da 100644 --- a/ue4cli/Utility.py +++ b/ue4cli/Utility.py @@ -1,4 +1,5 @@ -import os, platform, shlex, subprocess, sys +from .UtilityException import UtilityException +import os, platform, shlex, subprocess, sys, shutil class CommandOutput(object): """ @@ -21,23 +22,39 @@ def printStderr(*args, **kwargs): Prints to stderr instead of stdout """ if os.environ.get('UE4CLI_QUIET', '0') != '1': - print(*args, file=sys.stderr, **kwargs) + print('(ue4cli)', *args, file=sys.stderr, **kwargs) @staticmethod def readFile(filename): """ Reads data from a file """ - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + try: + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + except OSError as e: + raise UtilityException(f'failed to read file "{str(filename)}" due to: ({type(e).__name__}) {str(e)}') from e @staticmethod def writeFile(filename, data): """ Writes data to a file """ - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + try: + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + except OSError as e: + raise UtilityException(f'failed to write file "{str(filename)}" due to: ({type(e).__name__}) {str(e)}') from e + + @staticmethod + def moveFile(src, dst): + """ + Moves file from 'src' to 'dst' + """ + try: + shutil.move(src, dst) + except OSError as e: + raise UtilityException(f'failed to move file from "{str(src)}" to "{str(dst)}" due to: ({type(e).__name__}) {str(e)}') from e @staticmethod def patchFile(filename, replacements): @@ -52,6 +69,26 @@ def patchFile(filename, replacements): Utility.writeFile(filename, patched) + @staticmethod + def removeDir(path, ignore_errors=False): + """ + Recursively remove directory tree + """ + try: + shutil.rmtree(path, ignore_errors) + except OSError as e: + raise UtilityException(f'failed to remove directory "{str(path)}" due to: ({type(e).__name__}) {str(e)}') from e + + @staticmethod + def makeDirs(name, mode=0o777, exist_ok=False): + """ + Makes directory + """ + try: + os.makedirs(name, mode, exist_ok) + except OSError as e: + raise UtilityException(f'failed to create directory "{str(name)}" due to: ({type(e).__name__}) {str(e)}') from e + @staticmethod def forwardSlashes(paths): """ @@ -123,12 +160,12 @@ def capture(command, input=None, cwd=None, shell=False, raiseOnError=False): # If the child process failed and we were asked to raise an exception, do so if raiseOnError == True and proc.returncode != 0: - raise Exception( - 'child process ' + str(command) + - ' failed with exit code ' + str(proc.returncode) + - '\nstdout: "' + stdout + '"' + - '\nstderr: "' + stderr + '"' - ) + Utility.printStderr("Warning: child process failure encountered!") + Utility.printStderr("printing stdout..") + print(stdout) + Utility.printStderr("printing stderr..") + print(stderr) + raise UtilityException(f'child process {str(command)} failed with exit code {str(proc.returncode)}') return CommandOutput(proc.returncode, stdout, stderr) @@ -143,7 +180,7 @@ def run(command, cwd=None, shell=False, raiseOnError=False): returncode = subprocess.call(command, cwd=cwd, shell=shell) if raiseOnError == True and returncode != 0: - raise Exception('child process ' + str(command) + ' failed with exit code ' + str(returncode)) + raise UtilityException(f'child process {str(command)} failed with exit code {str(returncode)}') return returncode @staticmethod @@ -152,4 +189,4 @@ def _printCommand(command): Prints a command if verbose output is enabled """ if os.environ.get('UE4CLI_VERBOSE', '0') == '1': - Utility.printStderr('[UE4CLI] EXECUTE COMMAND:', command) + Utility.printStderr('EXECUTE COMMAND:', command) diff --git a/ue4cli/UtilityException.py b/ue4cli/UtilityException.py new file mode 100644 index 0000000..53af630 --- /dev/null +++ b/ue4cli/UtilityException.py @@ -0,0 +1,2 @@ +class UtilityException(Exception): + pass diff --git a/ue4cli/cli.py b/ue4cli/cli.py index d5f51e3..df88040 100644 --- a/ue4cli/cli.py +++ b/ue4cli/cli.py @@ -2,7 +2,9 @@ from .PluginManager import PluginManager from .UnrealManagerException import UnrealManagerException from .UnrealManagerFactory import UnrealManagerFactory -import os, sys +from .Utility import Utility +from .UtilityException import UtilityException +import os, sys, logging # Our list of supported commands SUPPORTED_COMMANDS = { @@ -200,8 +202,10 @@ def displayHelp(): print() def main(): + + logger = logging.getLogger(__name__) + try: - # Perform plugin detection and register our detected plugins plugins = PluginManager.getPlugins() for command in plugins: @@ -221,8 +225,22 @@ def main(): if command in SUPPORTED_COMMANDS: SUPPORTED_COMMANDS[command]['action'](manager, args) else: - raise UnrealManagerException('unrecognised command "' + command + '"') - - except UnrealManagerException as e: - print('Error: ' + str(e)) + # FIXME: This is the only place outside of UnrealManager... classes where we use UnrealManagerException. + # Not worth to create new Exception class for only one single case, at least not now. + raise UnrealManagerException(f'unrecognised command "{str(command)}"') from None + + except ( + UnrealManagerException, + UtilityException, + KeyboardInterrupt, + ) as e: + Utility.printStderr(f'Error: ({type(e).__name__}) {str(e)}') sys.exit(1) + + except BaseException as e: + Utility.printStderr('Unhandled exception! Crashing...') + logging.basicConfig(level=logging.DEBUG) + logger.exception(e) + Utility.printStderr('ue4cli has crashed! Please, report it at: https://github.com/adamrehn/ue4cli/issues') + sys.exit(1) +