Actually running validations

vontell · vontell · commit bb38d2c003cc · 2024-12-16T15:12:13.000-05:00
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/AndKeyFrameCriteriaEvaluator.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/AndKeyFrameCriteriaEvaluator.cs
@@ -6,13 +6,13 @@ namespace RegressionGames.StateRecorder.BotSegments
 {
     public static class AndKeyFrameCriteriaEvaluator
     {
-        public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, KeyFrameCriteria criteria)
+        public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, KeyFrameCriteria criteria)
         {
             if (criteria.data is AndKeyFrameCriteriaData { criteriaList: not null } andCriteria)
             {
                 try
                 {
-                    return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, BooleanCriteria.And, andCriteria.criteriaList);
+                    return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, BooleanCriteria.And, andCriteria.criteriaList);
                 }
                 catch (Exception)
                 {
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs
@@ -490,6 +490,7 @@ private void EvaluateBotSegments()
                         nextBotSegmentIndex == 0,
                         nextBotSegment.Replay_SegmentNumber,
                         nextBotSegment.Replay_ActionCompleted,
+                        nextBotSegment.Replay_ValidationsCompleted,
                         nextBotSegment.endCriteria
                     );
 
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/JsonConverters/KeyFrameCriteriaJsonConverter.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/JsonConverters/KeyFrameCriteriaJsonConverter.cs
@@ -43,6 +43,9 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
                 case KeyFrameCriteriaType.ActionComplete:
                     data = jObject["data"].ToObject<ActionCompleteKeyFrameCriteriaData>(serializer);
                     break;
+                case KeyFrameCriteriaType.ValidationsComplete:
+                    data = jObject["data"].ToObject<ValidationsCompleteKeyFrameCriteriaData>(serializer);
+                    break;
                 case KeyFrameCriteriaType.CVObjectDetection:
                     data = jObject["data"].ToObject<CVObjectDetectionKeyFrameCriteriaData>(serializer);
                     break;
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/KeyFrameEvaluator.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/KeyFrameEvaluator.cs
@@ -81,10 +81,10 @@ public void PersistPriorFrameStatus()
         /**
          * <summary>Publicly callable.. caches the statuses of the last passed key frame for computing delta counts from</summary>
          */
-        public bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, List<KeyFrameCriteria> criteriaList)
+        public bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, List<KeyFrameCriteria> criteriaList)
         {
             _newUnmatchedCriteria.Clear();
-            bool matched = MatchedHelper(firstSegment, segmentNumber, botActionCompleted, BooleanCriteria.And, criteriaList);
+            bool matched = MatchedHelper(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, BooleanCriteria.And, criteriaList);
             if (matched)
             {
                 CVTextCriteriaEvaluator.Cleanup(segmentNumber);
@@ -106,7 +106,7 @@ public bool Matched(bool firstSegment, int segmentNumber, bool botActionComplete
         /**
          * <summary>Only to be called internally by KeyFrameEvaluator. firstSegment represents if this is the first segment in the current pass's list of segments to evaluate</summary>
          */
-        internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botActionCompleted, BooleanCriteria andOr, List<KeyFrameCriteria> criteriaList)
+        internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, BooleanCriteria andOr, List<KeyFrameCriteria> criteriaList)
         {
             var objectFinders = Object.FindObjectsByType<ObjectFinder>(FindObjectsSortMode.None);
             var currentFrameCount = Time.frameCount;
@@ -194,6 +194,13 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction
                             return false;
                         }
                         break;
+                    case KeyFrameCriteriaType.ValidationsComplete:
+                        if (!validationsCompleted)
+                        {
+                            _newUnmatchedCriteria.Add("Waiting for validations to complete...");
+                            return false;
+                        }
+                        break;
                 }
             }
 
@@ -368,7 +375,7 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction
                 for (var j = 0; j < orCount; j++)
                 {
                     var orEntry = orsToMatch[j];
-                    var m = OrKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, orEntry);
+                    var m = OrKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, orEntry);
                     if (m)
                     {
                         if (andOr == BooleanCriteria.Or)
@@ -394,7 +401,7 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction
                 for (var j = 0; j < andCount; j++)
                 {
                     var andEntry = andsToMatch[j];
-                    var m = AndKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, andEntry);
+                    var m = AndKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, andEntry);
                     if (m)
                     {
                         if (andOr == BooleanCriteria.Or)
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotCriteria/ValidationsCompleteKeyFrameCriteriaData.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotCriteria/ValidationsCompleteKeyFrameCriteriaData.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Text;
+using RegressionGames.StateRecorder.JsonConverters;
+
+namespace RegressionGames.StateRecorder.BotSegments.Models.BotCriteria
+{
+    [Serializable]
+    public class ValidationsCompleteKeyFrameCriteriaData : IKeyFrameCriteriaData
+    {
+        public int apiVersion = SdkApiVersion.VERSION_11;
+        public float timeout = 0.0f;
+
+        public void WriteToStringBuilder(StringBuilder stringBuilder)
+        {
+            stringBuilder.Append("{\"apiVersion\":");
+            IntJsonConverter.WriteToStringBuilder(stringBuilder, apiVersion);
+            stringBuilder.Append(",\"timeout\":");
+            StringJsonConverter.WriteToStringBuilder(stringBuilder, timeout.ToString());
+            stringBuilder.Append("}");
+        }
+
+        public override string ToString()
+        {
+            StringBuilder sb = new StringBuilder(100);
+            WriteToStringBuilder(sb);
+            return sb.ToString();
+        }
+
+        public int EffectiveApiVersion()
+        {
+            return apiVersion;
+        }
+    }
+}
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotCriteria/ValidationsCompleteKeyFrameCriteriaData.cs.meta b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotCriteria/ValidationsCompleteKeyFrameCriteriaData.cs.meta
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotSegment.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/BotSegment.cs
@@ -71,6 +71,9 @@ public class BotSegment
         // Replay only - tracks if we have completed the action for this bot segment
         // returns true if botAction.IsCompleted || botAction.IsCompleted==null && Replay_Matched
         public bool Replay_ActionCompleted => botAction == null || (botAction.IsCompleted ?? Replay_Matched);
+        
+        // Replay only - true if we have completed the validations for this bot segment
+        public bool Replay_ValidationsCompleted => validations.Count == 0 || validations.All(v => v.HasSetAllResults());
 
         public void OnGUI(Dictionary<long, ObjectStatus> currentTransforms, Dictionary<long, ObjectStatus> currentEntities)
         {
@@ -142,6 +145,8 @@ public void PauseAction()
 
         public void ProcessValidation()
         {
+            // Go through each validation and process them. If they have not been started yet, this will also
+            // start them.
             foreach (var validation in validations)
             {
                 validation.ProcessValidation(Replay_SegmentNumber);
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/IRGSegmentValidationData.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/IRGSegmentValidationData.cs
@@ -39,12 +39,17 @@ public interface IRGSegmentValidationData
          */
         public void ResetResults();
 
+        /**
+         * Returns true if there are no "UNKNOWN" validations.
+         */
+        public bool HasSetAllResults();
+
         /**
          * Returns all results for this particular validation. In some
          * cases, this can be a set of results rather than just a single
          * result.
          */
-        public SegmentValidationResultContainer GetResults();
+        public SegmentValidationResultSetContainer GetResults();
         
         public void WriteToStringBuilder(StringBuilder stringBuilder);
         
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/KeyFrameCriteriaType.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/KeyFrameCriteriaType.cs
@@ -11,6 +11,7 @@ public enum KeyFrameCriteriaType
         CVImage,
         ActionComplete,
         CVObjectDetection,
+        ValidationsComplete
 
         //FUTURE
         //Path,
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidation.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidation.cs
@@ -23,11 +23,16 @@ public class SegmentValidation
         public void PrepareValidation(int segmentNumber)
         {
             data.PrepareValidation(segmentNumber);
+            _hasStarted = true;
         }
         
         // called once per frame
         public void ProcessValidation(int segmentNumber)
         {
+            if (!_hasStarted)
+            {
+                PrepareValidation(segmentNumber);
+            }
             data.ProcessValidation(segmentNumber);
         }
 
@@ -59,6 +64,11 @@ public void ReplayReset()
             data.ResetResults();
         }
 
+        public bool HasSetAllResults()
+        {
+            return data.HasSetAllResults();
+        }
+
         public void WriteToStringBuilder(StringBuilder stringBuilder)
         {
             stringBuilder.Append("{\"type\":");
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/ScriptSegmentValidationData.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/ScriptSegmentValidationData.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
 using System.Text;
 using System.Threading;
@@ -169,10 +170,15 @@ public void ResetResults()
         {
             _myValidationBehaviour?.ResetResults();
         }
+        
+        public bool HasSetAllResults()
+        {
+            return _myValidationBehaviour?.GetResults().validationResults.All(v => v.result != SegmentValidationStatus.UNKNOWN) ?? false;
+        }
 
-        public SegmentValidationResultContainer GetResults()
+        public SegmentValidationResultSetContainer GetResults()
         {
-            throw new System.NotImplementedException();
+            return _myValidationBehaviour?.GetResults();
         }
 
         public void WriteToStringBuilder(StringBuilder stringBuilder)
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/OrKeyFrameCriteriaEvaluator.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/OrKeyFrameCriteriaEvaluator.cs
@@ -6,13 +6,13 @@ namespace RegressionGames.StateRecorder.BotSegments
 {
     public static class OrKeyFrameCriteriaEvaluator
     {
-        public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, KeyFrameCriteria criteria)
+        public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, KeyFrameCriteria criteria)
         {
             if (criteria.data is OrKeyFrameCriteriaData { criteriaList: not null } orCriteria)
             {
                 try
                 {
-                    return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, BooleanCriteria.Or, orCriteria.criteriaList);
+                    return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, BooleanCriteria.Or, orCriteria.criteriaList);
                 }
                 catch (Exception)
                 {
diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateBehaviour.cs b/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateBehaviour.cs
@@ -88,7 +88,7 @@ public SegmentValidationResultSetContainer GetResults()
             return resultSet;
         }
 
-        public void ResetValidationStates()
+        public virtual void ResetValidationStates()
         {
             // The base implementation of this doesn't do anything - it's up to the validation script to implement this
         }
diff --git a/src/gg.regression.unity.bots/Tests/TestFramework/Scripts/RGTestUtils.cs b/src/gg.regression.unity.bots/Tests/TestFramework/Scripts/RGTestUtils.cs
@@ -197,6 +197,7 @@ public static IEnumerator WaitForBotCriteria(List<KeyFrameCriteria> botCriteria,
                     true,
                     0,
                     true,
+                    true,
                     botCriteria
                     );
 

Original file line number	Diff line number	Diff line change
`@@ -6,13 +6,13 @@ namespace RegressionGames.StateRecorder.BotSegments`
`6`	`6`	`{`
`7`	`7`	`public static class AndKeyFrameCriteriaEvaluator`
`8`	`8`	`{`
`9`		`- public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, KeyFrameCriteria criteria)`
	`9`	`+ public static bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, KeyFrameCriteria criteria)`
`10`	`10`	`{`
`11`	`11`	`if (criteria.data is AndKeyFrameCriteriaData { criteriaList: not null } andCriteria)`
`12`	`12`	`{`
`13`	`13`	`try`
`14`	`14`	`{`
`15`		`- return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, BooleanCriteria.And, andCriteria.criteriaList);`
	`15`	`+ return KeyFrameEvaluator.Evaluator.MatchedHelper(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, BooleanCriteria.And, andCriteria.criteriaList);`
`16`	`16`	`}`
`17`	`17`	`catch (Exception)`
`18`	`18`	`{`
Original file line number	Diff line number	Diff line change
`@@ -81,10 +81,10 @@ public void PersistPriorFrameStatus()`
`81`	`81`	`/**`
`82`	`82`	`* <summary>Publicly callable.. caches the statuses of the last passed key frame for computing delta counts from</summary>`
`83`	`83`	`*/`
`84`		`- public bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, List<KeyFrameCriteria> criteriaList)`
	`84`	`+ public bool Matched(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, List<KeyFrameCriteria> criteriaList)`
`85`	`85`	`{`
`86`	`86`	`_newUnmatchedCriteria.Clear();`
`87`		`- bool matched = MatchedHelper(firstSegment, segmentNumber, botActionCompleted, BooleanCriteria.And, criteriaList);`
	`87`	`+ bool matched = MatchedHelper(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, BooleanCriteria.And, criteriaList);`
`88`	`88`	`if (matched)`
`89`	`89`	`{`
`90`	`90`	`CVTextCriteriaEvaluator.Cleanup(segmentNumber);`
`@@ -106,7 +106,7 @@ public bool Matched(bool firstSegment, int segmentNumber, bool botActionComplete`
`106`	`106`	`/**`
`107`	`107`	`* <summary>Only to be called internally by KeyFrameEvaluator. firstSegment represents if this is the first segment in the current pass's list of segments to evaluate</summary>`
`108`	`108`	`*/`
`109`		`- internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botActionCompleted, BooleanCriteria andOr, List<KeyFrameCriteria> criteriaList)`
	`109`	`+ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botActionCompleted, bool validationsCompleted, BooleanCriteria andOr, List<KeyFrameCriteria> criteriaList)`
`110`	`110`	`{`
`111`	`111`	`var objectFinders = Object.FindObjectsByType<ObjectFinder>(FindObjectsSortMode.None);`
`112`	`112`	`var currentFrameCount = Time.frameCount;`
`@@ -194,6 +194,13 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction`
`194`	`194`	`return false;`
`195`	`195`	`}`
`196`	`196`	`break;`
	`197`	`+ case KeyFrameCriteriaType.ValidationsComplete:`
	`198`	`+ if (!validationsCompleted)`
	`199`	`+ {`
	`200`	`+ _newUnmatchedCriteria.Add("Waiting for validations to complete...");`
	`201`	`+ return false;`
	`202`	`+ }`
	`203`	`+ break;`
`197`	`204`	`}`
`198`	`205`	`}`
`199`	`206`
`@@ -368,7 +375,7 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction`
`368`	`375`	`for (var j = 0; j < orCount; j++)`
`369`	`376`	`{`
`370`	`377`	`var orEntry = orsToMatch[j];`
`371`		`- var m = OrKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, orEntry);`
	`378`	`+ var m = OrKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, orEntry);`
`372`	`379`	`if (m)`
`373`	`380`	`{`
`374`	`381`	`if (andOr == BooleanCriteria.Or)`
`@@ -394,7 +401,7 @@ internal bool MatchedHelper(bool firstSegment, int segmentNumber, bool botAction`
`394`	`401`	`for (var j = 0; j < andCount; j++)`
`395`	`402`	`{`
`396`	`403`	`var andEntry = andsToMatch[j];`
`397`		`- var m = AndKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, andEntry);`
	`404`	`+ var m = AndKeyFrameCriteriaEvaluator.Matched(firstSegment, segmentNumber, botActionCompleted, validationsCompleted, andEntry);`
`398`	`405`	`if (m)`
`399`	`406`	`{`
`400`	`407`	`if (andOr == BooleanCriteria.Or)`
Original file line number	Diff line number	Diff line change
`@@ -71,6 +71,9 @@ public class BotSegment`
`71`	`71`	`// Replay only - tracks if we have completed the action for this bot segment`
`72`	`72`	`// returns true if botAction.IsCompleted \|\| botAction.IsCompleted==null && Replay_Matched`
`73`	`73`	`public bool Replay_ActionCompleted => botAction == null \|\| (botAction.IsCompleted ?? Replay_Matched);`
	`74`	`+`
	`75`	`+ // Replay only - true if we have completed the validations for this bot segment`
	`76`	`+ public bool Replay_ValidationsCompleted => validations.Count == 0 \|\| validations.All(v => v.HasSetAllResults());`
`74`	`77`
`75`	`78`	`public void OnGUI(Dictionary<long, ObjectStatus> currentTransforms, Dictionary<long, ObjectStatus> currentEntities)`
`76`	`79`	`{`
`@@ -142,6 +145,8 @@ public void PauseAction()`
`142`	`145`
`143`	`146`	`public void ProcessValidation()`
`144`	`147`	`{`
	`148`	`+ // Go through each validation and process them. If they have not been started yet, this will also`
	`149`	`+ // start them.`
`145`	`150`	`foreach (var validation in validations)`
`146`	`151`	`{`
`147`	`152`	`validation.ProcessValidation(Replay_SegmentNumber);`
Original file line number	Diff line number	Diff line change
`@@ -23,11 +23,16 @@ public class SegmentValidation`
`23`	`23`	`public void PrepareValidation(int segmentNumber)`
`24`	`24`	`{`
`25`	`25`	`data.PrepareValidation(segmentNumber);`
	`26`	`+ _hasStarted = true;`
`26`	`27`	`}`
`27`	`28`
`28`	`29`	`// called once per frame`
`29`	`30`	`public void ProcessValidation(int segmentNumber)`
`30`	`31`	`{`
	`32`	`+ if (!_hasStarted)`
	`33`	`+ {`
	`34`	`+ PrepareValidation(segmentNumber);`
	`35`	`+ }`
`31`	`36`	`data.ProcessValidation(segmentNumber);`
`32`	`37`	`}`
`33`	`38`
`@@ -59,6 +64,11 @@ public void ReplayReset()`
`59`	`64`	`data.ResetResults();`
`60`	`65`	`}`
`61`	`66`
	`67`	`+ public bool HasSetAllResults()`
	`68`	`+ {`
	`69`	`+ return data.HasSetAllResults();`
	`70`	`+ }`
	`71`	`+`
`62`	`72`	`public void WriteToStringBuilder(StringBuilder stringBuilder)`
`63`	`73`	`{`
`64`	`74`	`stringBuilder.Append("{\"type\":");`