wiring more stuff up to use pcommandbatch

config/spinoffconfig.py

@@ -172,7 +172,6 @@
     'assets_phase_xcode',
     'ballistica_maya_tools.mel',
     'check_python_syntax',
-    'compile_python_file',
     'pcommand',
     'vmshell',
     'cloudshell',

src/assets/Makefile

@@ -150,7 +150,6 @@ ASSET_TARGETS_WIN_X64 += $(EXTRAS_TARGETS_WIN_X64)
 # Note: Code below needs updating when Python version changes (currently 3.11)
 define make-opt-pyc-target
 $1: $$(subst /__pycache__,,$$(subst .cpython-311.opt-1.pyc,.py,$1))
-#	@echo Compiling script: $$(subst $(BUILD_DIR)/,,$$^)
 	@$$(PCOMMANDBATCH) compile_python_file $$^
 endef
 
@@ -702,7 +701,6 @@ SCRIPT_TARGETS_PYC_PUBLIC = \
 # Rule to copy src asset scripts to dst.
 # (and make non-writable so I'm less likely to accidentally edit them there)
 $(SCRIPT_TARGETS_PY_PUBLIC) : $(BUILD_DIR)/%.py : %.py
-#	@echo Copying script: $(subst $(BUILD_DIR)/,,$@)
 	@$(PCOMMANDBATCH) copy_python_file $^ $@
 
 # These are too complex to define in a pattern rule;
@@ -781,7 +779,6 @@ SCRIPT_TARGETS_PYC_PUBLIC_TOOLS = \
 # Rule to copy src asset scripts to dst.
 # (and make non-writable so I'm less likely to accidentally edit them there)
 $(SCRIPT_TARGETS_PY_PUBLIC_TOOLS) : $(BUILD_DIR)/ba_data/python/%.py : $(TOOLS_DIR)/%.py
-#	@echo Copying script: $(subst $(BUILD_DIR)/,,$@)
 	@$(PCOMMANDBATCH) copy_python_file $^ $@
 
 # These are too complex to define in a pattern rule;
@@ -7423,6 +7420,21 @@ ba_data/%.tex2d.png : ../../.efrocachemap
 ba_data/%_+x.tex2d.png : ../../.efrocachemap
 	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
 
+ba_data/%_-x.tex2d.png: ../../.efrocachemap
+	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
+
+ba_data/%_+y.tex2d.png: ../../.efrocachemap
+	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
+
+ba_data/%_-y.tex2d.png: ../../.efrocachemap
+	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
+
+ba_data/%_+z.tex2d.png: ../../.efrocachemap
+	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
+
+ba_data/%_-z.tex2d.png: ../../.efrocachemap
+	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
+
 $(BUILD_DIR)/ba_data/%_preview.png : ../../.efrocachemap
 	@cd ../.. && tools/pcommand efrocache_get src/assets/$@
 

src/tools/pcommandbatch/pcommandbatch.c

@@ -24,6 +24,7 @@
 
 struct Context_ {
   const char* state_dir_path;
+  const char* project_dir_path;
   const char* instance_prefix;
   int instance_num;
   int pid;
@@ -46,9 +47,6 @@ int main(int argc, char** argv) {
   struct Context_ ctx;
   memset(&ctx, 0, sizeof(ctx));
 
-  ctx.state_dir_path = NULL;
-  ctx.instance_prefix = NULL;
-  ctx.pcommandpath = NULL;
   ctx.server_idle_seconds = 5;
   ctx.pid = getpid();
 
@@ -221,10 +219,11 @@ int establish_connection_(const struct Context_* ctx) {
       }
       char buf[512];
       snprintf(buf, sizeof(buf),
-               "%s run_pcommandbatch_server --timeout %d --state-dir %s "
-               "--instance %s_%d %s",
-               ctx->pcommandpath, ctx->server_idle_seconds, ctx->state_dir_path,
-               ctx->instance_prefix, ctx->instance_num, endbuf);
+               "%s run_pcommandbatch_server --timeout %d --project-dir %s"
+               " --state-dir %s --instance %s_%d %s",
+               ctx->pcommandpath, ctx->server_idle_seconds, ctx->project_dir_path,
+               ctx->state_dir_path, ctx->instance_prefix, ctx->instance_num,
+               endbuf);
       system(buf);
 
       // Spin and wait up to a few seconds for the file to appear.
@@ -327,12 +326,14 @@ int calc_paths_(struct Context_* ctx) {
   // this is project-root and src/assets
   if (path_exists_("config/projectconfig.json")) {
     // Looks like we're in project root.
+    ctx->project_dir_path = ".";
     ctx->state_dir_path = ".cache/pcommandbatch";
     ctx->instance_prefix = "root";
     ctx->pcommandpath = "tools/pcommand";
   } else if (path_exists_("ba_data")
              && path_exists_("../../config/projectconfig.json")) {
     // Looks like we're in src/assets.
+    ctx->project_dir_path = "../..";
     ctx->state_dir_path = "../../.cache/pcommandbatch";
     ctx->instance_prefix = "assets";
     ctx->pcommandpath = "../../tools/pcommand";
@@ -362,16 +363,27 @@ int calc_paths_(struct Context_* ctx) {
   // instance that spins up worker instances as needed. Though such a fancy
   // setup might be overkill.
   ctx->instance_num = rand() % 6;
+
+  // I was wondering if using pid would lead to a more even distribution,
+  // but it didn't make a significant difference in my tests. And I worry
+  // there would be some odd corner case where pid isn't going up evenly, so
+  // sticking with rand() for now. ctx->instance_num = ctx->pid % 6;
   return 0;
 }
 
 int send_command_(struct Context_* ctx, int argc, char** argv) {
   // Build a json array of our args.
+  cJSON* req = cJSON_CreateObject();
   cJSON* array = cJSON_CreateArray();
   for (int i = 0; i < argc; ++i) {
     cJSON_AddItemToArray(array, cJSON_CreateString(argv[i]));
   }
-  char* json_out = cJSON_Print(array);
+  cJSON_AddItemToObject(req, "a", array);
+  cJSON_AddItemToObject(req, "t",
+                        isatty(1) ? cJSON_CreateTrue() : cJSON_CreateFalse());
+  cJSON_AddItemToObject(req, "t",
+                        isatty(1) ? cJSON_CreateTrue() : cJSON_CreateFalse());
+  char* json_out = cJSON_Print(req);
 
   // Send our command.
   int msglen = strlen(json_out);
@@ -393,7 +405,7 @@ int send_command_(struct Context_* ctx, int argc, char** argv) {
 
   // Clean up our mess after we've sent them on their way.
   free(json_out);
-  cJSON_Delete(array);
+  cJSON_Delete(req);
 
   return 0;
 }

tools/batools/assetsmakefile.py

@@ -265,7 +265,7 @@ def _get_py_targets_subset(
         '# (and make non-writable so I\'m less likely to '
         'accidentally edit them there)\n'
         f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
-        '#\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
+        # '#\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
         '\t@$(PCOMMANDBATCH) copy_python_file $^ $@\n'
     )
 
@@ -395,10 +395,11 @@ def _get_extras_targets_win(
         f'# __EFROCACHE_TARGET__\n'
         f'$(EXTRAS_TARGETS_WIN_{p_up}) : $(BUILD_DIR)/% :'
         ' %\n'
-        '\t@echo Copying file: $(subst $(BUILD_DIR)/,,$@)\n'
-        '\t@mkdir -p $(dir $@)\n'
-        '\t@rm -f $@\n'
-        '\t@cp $^ $@\n'
+        '\t@$(PCOMMANDBATCH) copy_win_extra_file $^ $@\n'
+        # '\t@echo Copying file: $(subst $(BUILD_DIR)/,,$@)\n'
+        # '\t@mkdir -p $(dir $@)\n'
+        # '\t@rm -f $@\n'
+        # '\t@cp $^ $@\n'
     )
 
     return out

tools/batools/pcommands.py

@@ -785,7 +785,7 @@ def stage_build() -> None:
 
     try:
         batools.staging.stage_build(
-            projroot=str(pcommand.PROJROOT), args=sys.argv[2:]
+            projroot=str(pcommand.PROJROOT), args=pcommand.get_args()
         )
     except CleanError as exc:
         exc.pretty_print()

tools/efrotools/pcommand.py

@@ -18,6 +18,8 @@
     import threading
     from typing import Any
 
+    from efro.terminal import ClrBase
+
 # Absolute path of the project root.
 PROJROOT = Path(__file__).resolve().parents[2]
 
@@ -76,6 +78,22 @@ def get_args() -> list[str]:
     return argv[2:]
 
 
+def clr() -> type[ClrBase]:
+    """Like efro.terminal.Clr but works correctly under pcommandbatch."""
+    import efro.terminal
+
+    # Note: currently just using the 'isatty' value from the client.
+    # ideally should expand the client-side logic to exactly match what
+    # efro.terminal.Clr does locally.
+    if _g_batch_server_mode:
+        assert _g_thread_local_storage is not None
+        isatty = _g_thread_local_storage.isatty
+        assert isinstance(isatty, bool)
+        return efro.terminal.ClrAlways if isatty else efro.terminal.ClrNever
+
+    return efro.terminal.Clr
+
+
 def set_output(output: str, newline: bool = True) -> None:
     """Set an output string for the current pcommand.
 
@@ -103,12 +121,6 @@ def _run_pcommand(sysargv: list[str]) -> int:
 
     assert _g_funcs is not None
 
-    # If we're in batch mode, stuff these args into our thread-local
-    # storage.
-    if _g_batch_server_mode:
-        assert _g_thread_local_storage is not None
-        _g_thread_local_storage.argv = sysargv
-
     retval = 0
     show_help = False
     if len(sysargv) < 2:
@@ -192,7 +204,9 @@ def is_batch() -> bool:
     return _g_batch_server_mode
 
 
-def run_client_pcommand(args: list[str], log_path: str) -> tuple[int, str]:
+def run_client_pcommand(
+    args: list[str], log_path: str, isatty: bool
+) -> tuple[int, str]:
     """Call a pcommand function when running as a batch server."""
     assert _g_batch_server_mode
     assert _g_thread_local_storage is not None
@@ -203,6 +217,11 @@ def run_client_pcommand(args: list[str], log_path: str) -> tuple[int, str]:
     if hasattr(_g_thread_local_storage, 'output'):
         delattr(_g_thread_local_storage, 'output')
 
+    # Stuff args into our thread-local storage so the user can get at
+    # them.
+    _g_thread_local_storage.argv = args
+    _g_thread_local_storage.isatty = isatty
+
     # Run the command.
     resultcode: int = _run_pcommand(args)
 
@@ -214,7 +233,8 @@ def run_client_pcommand(args: list[str], log_path: str) -> tuple[int, str]:
             output += '\n'
         output += (
             f'Error: pcommandbatch command failed: {args}.'
-            f" See '{log_path}' for more info.\n"
+            f" For more info, see '{log_path}', or rerun with"
+            ' env var BA_PCOMMANDBATCH_DISABLE=1 to see all output here.\n'
         )
 
     assert isinstance(output, str)