temp jobs are no longer rehashed everytime a previously failed depend…

…ency restarts

dev_utils/interactive_test.py

@@ -40,15 +40,16 @@ def do_a(of):
 ppg.FileGeneratingJob("A", do_a)
 
 
-def do_b(ofs):
-    assert ofs[0].name == 'b'
+def do_b(ofs, prefix):
+    assert ofs[0].name == "b"
     ofs[0].write_text("B")
 
 
-ppg.MultiFileGeneratingJob(["b", 'B1', 'shubidudb'], do_b)
+b = ppg.SharedMultiFileGeneratingJob("shi", ["b", "B1", "shubidudb"], do_b)
 
 
 c = ppg.FileGeneratingJob("C", do_a, resources=ppg.Resources.Exclusive)
+c.depends_on(b)
 
 
 ppg.run()

python/pypipegraph2/jobs.py

@@ -27,7 +27,8 @@
 from .util import (
     log_info,
     log_error,
-    log_warning,  # log_debug,
+    log_warning,
+    log_debug,
     log_trace,
     log_job_trace,
 )
@@ -1085,11 +1086,20 @@ def __init__(
         self._single_file = False
 
     def run(self, runner, historical_output):
+        log_error(f"running {self.job_id}")
         if historical_output and self.all_files_exist():
-            new_hashes = {
-                str(of): hashers.hash_file(self._map_filename(of))
-                for of in self.org_files
-            }
+            new_hashes = {}
+            for filename in self.org_files:
+                filename = str(filename)
+                if filename in historical_output:
+                    stat = Path(self._map_filename(filename)).stat()
+                    if int(stat.st_mtime) == historical_output[filename].get(
+                        "mtime", -1
+                    ) and stat.st_size == historical_output[filename].get("size", -1):
+                        new_hashes[filename] = historical_output[filename]
+                        continue
+                new_hashes[filename] = hashers.hash_file(self._map_filename(filename))
+
             all_ok = True
             for filename in self.org_files:
                 key = str(filename)
@@ -1099,10 +1109,11 @@ def run(self, runner, historical_output):
                 else:
                     all_ok = False
             if all_ok:
-                log_job_trace(
+                log_debug(
                     "Temp file output existed and had same hashes. No recalc, but job 'ran'"
                 )
                 return new_hashes
+
         return MultiFileGeneratingJob.run(self, runner, historical_output)
 
     def all_files_exist(self):

tests/test_jobs.py

@@ -1941,19 +1941,6 @@ def do_a(ofs):
         assert Path("a").read_text() == "4"
         assert Path("c").read_text() == "1"
 
-    def test_order_of_files_is_kept_for_callback(self):
-        def do_b(ofs):
-            assert ofs[0].name == "b"
-            assert ofs[1].name == "B1"
-            ofs[0].write_text("B")
-            ofs[1].write_text("B")
-
-        a = ppg.MultiTempFileGeneratingJob(["b", "B1"], do_b)
-        b = ppg.FileGeneratingJob("c", lambda of: of.write_text("b"))
-        b.depends_on(a)
-
-        ppg.run()
-
 
 @pytest.mark.usefixtures("create_out_dir")
 @pytest.mark.usefixtures("ppg2_per_test")
@@ -2030,6 +2017,50 @@ def test_non_iterable(self):
         with pytest.raises(TypeError):
             ppg.MultiTempFileGeneratingJob(25, lambda of: write("out/A", param))
 
+    def test_order_of_files_is_kept_for_callback(self):
+        def do_b(ofs):
+            assert ofs[0].name == "b"
+            assert ofs[1].name == "B1"
+            ofs[0].write_text("B")
+            ofs[1].write_text("B")
+
+        a = ppg.MultiTempFileGeneratingJob(["b", "B1"], do_b)
+        b = ppg.FileGeneratingJob("c", lambda of: of.write_text("b"))
+        b.depends_on(a)
+
+        ppg.run()
+
+    def test_no_reshash_on_fail_of_dep(self, mocker):
+        def gen():
+            def do_b(ofs):
+                ofs[0].write_text("b")
+                ofs[1].write_text("B1")
+
+            b = ppg.MultiTempFileGeneratingJob(["b", "B1"], do_b)
+
+            def do_c(of):
+                raise ValueError()
+
+            c = ppg.FileGeneratingJob("c", lambda of: do_c)
+            c.depends_on(b)
+
+        gen()
+        spy = mocker.spy(ppg.hashers, "hash_file")
+        with pytest.raises(ppg.JobsFailed):
+            ppg.run()
+        assert Path("b").exists()
+        assert Path("B1").exists()
+        assert not Path("c").exists()
+
+        till_here = spy.call_count
+
+        ppg.new()
+        gen()
+        with pytest.raises(ppg.JobsFailed):
+            ppg.run()
+        now = spy.call_count
+        assert now == till_here
+
 
 @pytest.mark.usefixtures("ppg2_per_test")
 class TestNoDotDotInJobIds: