diff --git a/Makefile b/Makefile index 048fe5803..086467993 100644 --- a/Makefile +++ b/Makefile @@ -266,10 +266,10 @@ prefab-mac-arm64-gui-debug-build: prereqs assets-cmake \ @$(STAGE_BUILD) -cmake -debug build/prefab/full/mac_arm64_gui/debug build/prefab/full/mac_%_gui/debug/ballisticakit: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/mac_%_gui/debug/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Mac gui release: @@ -296,10 +296,10 @@ prefab-mac-arm64-gui-release-build: prereqs assets-cmake \ @$(STAGE_BUILD) -cmake -release build/prefab/full/mac_arm64_gui/release build/prefab/full/mac_%_gui/release/ballisticakit: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/mac_%_gui/release/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Mac server debug: @@ -326,10 +326,10 @@ prefab-mac-arm64-server-debug-build: prereqs assets-server \ @$(STAGE_BUILD) -cmakeserver -debug build/prefab/full/mac_arm64_server/debug build/prefab/full/mac_%_server/debug/dist/ballisticakit_headless: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/mac_%_server/debug/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Mac server release: @@ -358,10 +358,10 @@ prefab-mac-arm64-server-release-build: prereqs assets-server \ build/prefab/full/mac_arm64_server/release build/prefab/full/mac_%_server/release/dist/ballisticakit_headless: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/mac_%_server/release/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Linux gui debug: @@ -388,10 +388,10 @@ prefab-linux-arm64-gui-debug-build: prereqs assets-cmake \ @$(STAGE_BUILD) -cmake -debug build/prefab/full/linux_arm64_gui/debug build/prefab/full/linux_%_gui/debug/ballisticakit: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/linux_%_gui/debug/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Linux gui release: @@ -418,10 +418,10 @@ prefab-linux-arm64-gui-release-build: prereqs assets-cmake \ @$(STAGE_BUILD) -cmake -release build/prefab/full/linux_arm64_gui/release build/prefab/full/linux_%_gui/release/ballisticakit: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/linux_%_gui/release/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Linux server debug: @@ -450,10 +450,10 @@ prefab-linux-arm64-server-debug-build: prereqs assets-server \ build/prefab/full/linux_arm64_server/debug build/prefab/full/linux_%_server/debug/dist/ballisticakit_headless: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/linux_%_server/debug/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Linux server release: @@ -482,10 +482,10 @@ prefab-linux-arm64-server-release-build: prereqs assets-server \ build/prefab/full/linux_arm64_server/release build/prefab/full/linux_%_server/release/dist/ballisticakit_headless: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/linux_%_server/release/libballisticaplus.a: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Windows gui debug: @@ -502,13 +502,13 @@ prefab-windows-x86-gui-debug-build: prereqs assets-windows-$(WINPLAT_X86) \ build/prefab/full/windows_x86_gui/debug build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Debug_%/BallisticaKitGenericPlus.lib: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Debug_%/BallisticaKitGenericPlus.pdb: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Windows gui release: @@ -526,13 +526,13 @@ prefab-windows-x86-gui-release-build: prereqs \ build/prefab/full/windows_x86_gui/release build/prefab/full/windows_x86_gui/release/BallisticaKit.exe: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Release_%/BallisticaKitGenericPlus.lib: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Release_%/BallisticaKitGenericPlus.pdb: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Windows server debug: @@ -551,13 +551,13 @@ prefab-windows-x86-server-debug-build: prereqs \ build/prefab/full/windows_x86_server/debug build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Debug_%/BallisticaKitHeadlessPlus.lib: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Debug_%/BallisticaKitHeadlessPlus.pdb: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Windows server release: @@ -576,13 +576,13 @@ prefab-windows-x86-server-release-build: prereqs \ build/prefab/full/windows_x86_server/release build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Release_%/BallisticaKitHeadlessPlus.lib: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ build/prefab/lib/windows/Release_%/BallisticaKitHeadlessPlus.pdb: .efrocachemap - @tools/pcommand efrocache_get $@ + @.cache/pcommandbatch/pcommandbatch efrocache_get $@ # Tell make which of these targets don't represent files. .PHONY: prefab-gui-debug prefab-gui-release prefab-gui-debug-build \ @@ -1222,10 +1222,6 @@ SKIP_ENV_CHECKS ?= 0 tools/pcommand checkenv && mkdir -p .cache && touch .cache/checkenv; \ fi -foof: CHANGELOG.md CONTRIBUTORS.md Makefile - echo OUT IS $@ - echo IN IS $^ - $(PCOMMANDBATCHBIN): src/tools/pcommandbatch/pcommandbatch.c \ src/tools/pcommandbatch/cJSON.c @tools/pcommand build_pcommandbatch $^ $@ diff --git a/src/assets/Makefile b/src/assets/Makefile index 4b1970c08..a84663b22 100644 --- a/src/assets/Makefile +++ b/src/assets/Makefile @@ -1757,7 +1757,7 @@ SCRIPT_TARGETS_PYC_PRIVATE_APPLE = \ # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) $(SCRIPT_TARGETS_PY_PRIVATE_APPLE) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # These are too complex to define in a pattern rule; # Instead we generate individual targets in a loop. @@ -2701,7 +2701,7 @@ SCRIPT_TARGETS_PYC_PRIVATE_ANDROID = \ # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) $(SCRIPT_TARGETS_PY_PRIVATE_ANDROID) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # These are too complex to define in a pattern rule; # Instead we generate individual targets in a loop. @@ -2759,7 +2759,7 @@ SCRIPT_TARGETS_PYC_PRIVATE_COMMON = \ # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) $(SCRIPT_TARGETS_PY_PRIVATE_COMMON) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # These are too complex to define in a pattern rule; # Instead we generate individual targets in a loop. @@ -3687,7 +3687,7 @@ SCRIPT_TARGETS_PYC_PRIVATE_WIN_WIN32 = \ # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) $(SCRIPT_TARGETS_PY_PRIVATE_WIN_WIN32) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # These are too complex to define in a pattern rule; # Instead we generate individual targets in a loop. @@ -4615,7 +4615,7 @@ SCRIPT_TARGETS_PYC_PRIVATE_WIN_X64 = \ # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) $(SCRIPT_TARGETS_PY_PRIVATE_WIN_X64) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # These are too complex to define in a pattern rule; # Instead we generate individual targets in a loop. @@ -7290,7 +7290,7 @@ EXTRAS_TARGETS_WIN_WIN32 = \ # Rule to copy src extras to build. $(EXTRAS_TARGETS_WIN_WIN32) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ EXTRAS_TARGETS_WIN_X64 = \ @@ -7380,7 +7380,7 @@ EXTRAS_TARGETS_WIN_X64 = \ # Rule to copy src extras to build. $(EXTRAS_TARGETS_WIN_X64) : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # __AUTOGENERATED_PRIVATE_END__ @@ -7388,64 +7388,64 @@ ASSET_TARGETS_COMMON_GUI += $(MESH_TARGETS) $(COLLISION_MESH_TARGETS) ASSET_TARGETS_COMMON_SERVER += $(COLLISION_MESH_TARGETS) $(BUILD_DIR)/%.bob : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.cob : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.ogg : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.fdata : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.pem : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # Langdata one-off json file. $(BUILD_DIR)/ba_data/data/langdata.json : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # Languages json files. $(BUILD_DIR)/ba_data/data/languages/%.json : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ # Map json files. $(BUILD_DIR)/ba_data/data/maps/%.json : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%.tex2d.png : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_+x.tex2d.png : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_-x.tex2d.png: ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_+y.tex2d.png: ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_-y.tex2d.png: ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_+z.tex2d.png: ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ ba_data/%_-z.tex2d.png: ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/ba_data/%_preview.png : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.dds : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.pvr : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ $(BUILD_DIR)/%.ktx : ../../.efrocachemap - @cd ../.. && tools/pcommand efrocache_get src/assets/$@ + @cd ../.. && .cache/pcommandbatch/pcommandbatch efrocache_get src/assets/$@ audio: $(AUDIO_TARGETS) audio-clean: diff --git a/tools/batools/pcommands.py b/tools/batools/pcommands.py index 0b2d8a292..dc2d8a130 100644 --- a/tools/batools/pcommands.py +++ b/tools/batools/pcommands.py @@ -475,11 +475,13 @@ def efrocache_get() -> None: """Get a file from efrocache.""" from efrotools.efrocache import get_target - pcommand.disallow_in_batch() - - if len(sys.argv) != 3: + args = pcommand.get_args() + if len(args) != 1: raise RuntimeError('Expected exactly 1 arg') - get_target(sys.argv[2]) + + output = get_target(args[0], batch=pcommand.is_batch(), clr=pcommand.clr()) + if pcommand.is_batch(): + pcommand.set_output(output) def get_modern_make() -> None: diff --git a/tools/batools/pcommands2.py b/tools/batools/pcommands2.py index 3dee371f8..69236812d 100644 --- a/tools/batools/pcommands2.py +++ b/tools/batools/pcommands2.py @@ -197,6 +197,7 @@ def clean_orphaned_assets() -> None: def win_ci_install_prereqs() -> None: """Install bits needed for basic win ci.""" import json + from efrotools.efrocache import get_target pcommand.disallow_in_batch() @@ -230,7 +231,7 @@ def win_ci_install_prereqs() -> None: needed_targets.add(target) for target in needed_targets: - get_target(target) + get_target(target, batch=pcommand.is_batch(), clr=pcommand.clr()) def win_ci_binary_build() -> None: diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index d2f7306b0..9a51b2758 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -29,7 +29,8 @@ from efro.terminal import Clr if TYPE_CHECKING: - pass + import efro.terminal + TARGET_TAG = '# __EFROCACHE_TARGET__' @@ -112,14 +113,17 @@ def _project_centric_path(path: str) -> str: return abspath[len(projpath) :] -def get_target(path: str) -> None: +def get_target(path: str, batch: bool, clr: type[efro.terminal.ClrBase]) -> str: """Fetch a target path from the cache, downloading if need be.""" # pylint: disable=too-many-locals # pylint: disable=too-many-statements + # pylint: disable=too-many-branches import tempfile from efro.error import CleanError + output_lines: list[str] = [] + local_cache_dir = get_local_cache_dir() path = _project_centric_path(path) @@ -150,8 +154,12 @@ def get_target(path: str) -> None: existing_hash = get_existing_file_hash(path) if existing_hash == hashval: os.utime(path, None) - print(f'Refreshing from cache: {path}') - return + msg = f'Refreshing from cache: {path}' + if batch: + output_lines.append(msg) + else: + print(msg) + return '\n'.join(output_lines) # Ok we need to download the cache file. # Ok there's not a valid file in place already. Clear out whatever @@ -163,7 +171,11 @@ def get_target(path: str) -> None: if not os.path.exists(local_cache_path): with tempfile.TemporaryDirectory() as tmpdir: local_cache_dl_path = os.path.join(tmpdir, 'dl') - print(f'Downloading: {Clr.BLU}{path}{Clr.RST}') + msg = f'Downloading: {clr.BLU}{path}{clr.RST}' + if batch: + output_lines.append(msg) + else: + print(msg) result = subprocess.run( [ 'curl', @@ -203,7 +215,11 @@ def get_target(path: str) -> None: # Ok we should have a valid file in our cache dir at this point. # Just expand it to the target path. - print(f'Extracting: {path}') + msg = f'Extracting: {path}' + if batch: + output_lines.append(msg) + else: + print(msg) # Extract and stage the file in a temp dir before doing # a final move to the target location to be as atomic as possible. @@ -233,6 +249,8 @@ def get_target(path: str) -> None: if not os.path.exists(path): raise RuntimeError(f'File {path} did not wind up as expected.') + return '\n'.join(output_lines) + def filter_makefile(makefile_dir: str, contents: str) -> str: """Filter makefile contents to use efrocache lookups.""" @@ -632,33 +650,38 @@ def warm_start_cache() -> None: # server system of some sort so we don't have to spin up a full # Python process for each and every file we need to touch. In that # case, this optimization would probably be unnecessary. - cachemap: dict[str, str] - with open(CACHE_MAP_NAME, encoding='utf-8') as infile: - cachemap = json.loads(infile.read()) - assert isinstance(cachemap, dict) - cachemap_mtime = os.path.getmtime(CACHE_MAP_NAME) - entries: list[tuple[str, str]] = [] - for fname, filehash in cachemap.items(): - # File hasn't been pulled from cache yet = ignore. - if not os.path.exists(fname): - continue - - # File is newer than the cache map = ignore. - if cachemap_mtime < os.path.getmtime(fname): - continue - - # Don't have the cache source file for this guy = ignore. This - # can happen if cache files have been blown away since the last - # time this was built. - cachefile = os.path.join(local_cache_dir, _path_from_hash(filehash)) - if not os.path.exists(cachefile): - continue - - # Ok, add it to the list of files we can potentially update - # timestamps on once we check its hash. - entries.append((fname, filehash)) - - if entries: - # Now fire off a multithreaded executor to check hashes and - # update timestamps. - _check_warm_start_entries(entries) + # + # UPDATE - we now have that lightweight build system (pcommandbatch) + # which means individual refreshes are now much less expensive than + # before, so disabling this for now. + if bool(False): + cachemap: dict[str, str] + with open(CACHE_MAP_NAME, encoding='utf-8') as infile: + cachemap = json.loads(infile.read()) + assert isinstance(cachemap, dict) + cachemap_mtime = os.path.getmtime(CACHE_MAP_NAME) + entries: list[tuple[str, str]] = [] + for fname, filehash in cachemap.items(): + # File hasn't been pulled from cache yet = ignore. + if not os.path.exists(fname): + continue + + # File is newer than the cache map = ignore. + if cachemap_mtime < os.path.getmtime(fname): + continue + + # Don't have the cache source file for this guy = ignore. This + # can happen if cache files have been blown away since the last + # time this was built. + cachefile = os.path.join(local_cache_dir, _path_from_hash(filehash)) + if not os.path.exists(cachefile): + continue + + # Ok, add it to the list of files we can potentially update + # timestamps on once we check its hash. + entries.append((fname, filehash)) + + if entries: + # Now fire off a multithreaded executor to check hashes and + # update timestamps. + _check_warm_start_entries(entries)