Enabled requirements aggregation for Serenity/JS (#3364)

* Reduced number of static method calls in AggregateRequirements

* Corrected a typo

* Removed another static call with an instantiated equivalent

* Make AggregateRequirements dependency on EnvironmentVariables explicit so that they can be mocked in the test

* Support for aggregating requirements from Serenity/JS reports

* Code clean-up

* Removed unused imports

* Added a missing test

* Corrected regular expression pattern escaping

* Corrected RequirementAncestry so that top-level requirements are compatible with how TestOutcomesTagProvider generates them and so that their equals comparison works

* Corrected ReportNameProvider so that links to requirement summaries are compatible with those generated for requirements tags

* Theme, capability and feature-level reports show up correctly

* Disable parallel processing in AggregateRequirementsService as it has some bug that prevents requirements from being correctly merged

* Derive the name of the requirement from pathElement instead of the human-readable name as the display name is already human-readable

* Added missing tests to prevent future regressions in RequirementAncestryTest

* Use the human-readable name as displayName for requirements read from the file system

* Updated tests to correctly assert on displayName or name

* Check both the name and the displayName of the requirement

* Fixed thread safety issues and enabled parallel requirement processing

* Introduced SpecFileFilters to reduce code duplication

serenity-cli/src/test/groovy/net/serenitybdd/cli/WhenLoadingRequirementsFromAProvidedDIrectoryStructure.groovy

@@ -13,6 +13,7 @@ class WhenLoadingRequirementsFromAProvidedDirectoryStructure extends Specificati
             def tagProvider = new FileSystemRequirementsTagProvider(environmentVariables,
                                                                     "src/test/resources/featuredir")
         then:
-            tagProvider.requirements.collect {it.name } as Set == ["Maintain my todo list","Record todos"] as Set
+            tagProvider.requirements.collect {it.name } as Set == ["maintain_my_todo_list","record_todos"] as Set
+            tagProvider.requirements.collect {it.displayName } as Set == ["Maintain my todo list","Record todos"] as Set
     }
 }

serenity-core/src/test/groovy/net/thucydides/core/requirements/WhenGeneratingRequirementsReportData.groovy

@@ -27,7 +27,7 @@ class WhenGeneratingRequirementsReportData extends Specification {
 
     List<RequirementsTagProvider> requirementsProviders
     ReportNameProvider reportNameProvider
-    
+
     def setup() {
         FeatureCache.getCache().close()
         def vars = new MockEnvironmentVariables()
@@ -49,10 +49,10 @@ class WhenGeneratingRequirementsReportData extends Specification {
             RequirementsOutcomes outcomes = requirmentsOutcomeFactory.buildRequirementsOutcomesFrom(noTestOutcomes)
         then: "all the known capabilities should be listed"
             def requirementsNames = outcomes.requirementOutcomes.collect {it.requirement.name}
-            requirementsNames == ["Grow cucumbers", "Grow potatoes", "Grow wheat",  "Raise chickens", "Apples", "Nice zucchinis", "Potatoes"]
+            requirementsNames == ["grow cucumbers", "Grow potatoes", "Grow wheat",  "raise_chickens", "Apples", "Nice zucchinis", "Potatoes"]
         and: "the display name should be obtained from the narrative file where present"
             def requirementsDisplayNames = outcomes.requirementOutcomes.collect {it.requirement.displayName}
-            requirementsDisplayNames == ["Grow cucumbers", "Grow lots of potatoes", "Grow wheat", "Raise chickens", "Apples", "Nice zucchinis", "Potatoes title"]
+            requirementsDisplayNames == ["Grow cucumbers", "Grow potatoes", "Grow wheat", "Raise chickens", "Apples", "Nice zucchinis", "Potatoes title"]
     }
 
     def "should report no test results for requirements without associated tests"() {

serenity-core/src/test/groovy/net/thucydides/core/requirements/WhenLoadingRequirementDefinitionDescriptionFromADirectory.groovy

@@ -70,7 +70,7 @@ class WhenLoadingRequirementDefinitionDescriptionFromADirectory extends Specific
         then: "the narrativeText should be found"
             narrative.present
         and: "the narrativeText title and description should be loaded"
-            narrative.get().title.get() == "Grow more apples"
+            narrative.get().title.get() == "Grow apples"
             narrative.get().text.contains("In order to make apple pies") &&
             narrative.get().text.contains("As a farmer") &&
             narrative.get().text.contains("I want to grow apples")

serenity-core/src/test/groovy/net/thucydides/core/requirements/WhenLoadingRequirementOutcomesFromTheFileSystem.groovy

@@ -20,7 +20,7 @@ class WhenLoadingRequirementOutcomesFromTheFileSystem extends Specification {
         Requirements requirements = new FileSystemRequirements("sample-story-directories/capabilities_and_features")
         when: "We load the available requirements"
         def capabilities = requirements.requirementsService.requirements
-        def capabilityNames = capabilities.collect { it.name }
+        def capabilityNames = capabilities.collect { it.displayName }
         def capabilityTypes = capabilities.collect { it.type }
         then: "the requirements should be loaded from the first-level sub-directories"
         capabilityNames == ["Grow apples", "Grow potatoes", "Grow zuchinnis"]
@@ -41,7 +41,7 @@ class WhenLoadingRequirementOutcomesFromTheFileSystem extends Specification {
         RequirementsTagProvider capabilityProvider = new FileSystemRequirementsTagProvider("sample-story-directories/capabilities_and_features with spaces")
         when: "We load the available requirements"
         def capabilities = capabilityProvider.getRequirements()
-        def capabilityNames = capabilities.collect { it.name }
+        def capabilityNames = capabilities.collect { it.displayName }
         then: "the requirements should be loaded from the first-level sub-directories"
         capabilityNames == ["Grow apples", "Grow potatoes", "Grow zuchinnis"]
     }
@@ -51,14 +51,13 @@ class WhenLoadingRequirementOutcomesFromTheFileSystem extends Specification {
         RequirementsTagProvider capabilityProvider = new FileSystemRequirementsTagProvider("stories")
         when: "We load the available requirements"
         def capabilities = capabilityProvider.getRequirements()
-        def capabilityNames = capabilities.collect { it.name }
+        def capabilityNames = capabilities.collect { it.displayName }
         then: "the requirements should be loaded from the first-level sub-directories"
         capabilityNames == ["Grow cucumbers", "Grow potatoes", "Grow wheat", "Raise chickens"]
         and: "the child requirements should be found"
         def growCucumbers = capabilities.get(0)
         def cucumberFeatures = growCucumbers.children.collect { it.name }
         cucumberFeatures == ["Grow green cucumbers"]
-
     }
 
     def "Should be able to load issues from the default directory structure"() {

serenity-core/src/test/resources/sample-story-directories/capabilities_and_features with spaces/grow apples/narrative.txt

@@ -1,8 +1,8 @@
-Grow more apples
+Grow apples
 Meta:
 @issue #123
 
 Narrative:
 In order to make apple pies
 As a farmer
-I want to grow apples
+I want to grow apples

serenity-core/src/test/resources/sample-story-directories/capabilities_and_features/grow_apples/narrative.txt

@@ -1,8 +1,8 @@
-Grow more apples
+Grow apples
 Meta:
 @issue #123
 
 Narrative:
 In order to make apple pies
 As a farmer
-I want to grow apples
+I want to grow apples

serenity-core/src/test/resources/sample-story-directories/capabilities_and_features/grow_potatoes/narrative.txt

@@ -1,6 +1,6 @@
-Theme: Grow Potatoes
+Theme: Grow potatoes
 
 Narrative:
 In order to let my country eat chips
 As a farmer
-I want to grow potatoes
+I want to grow potatoes

serenity-core/src/test/resources/stories/grow_potatoes/narrative.txt

@@ -1,6 +1,6 @@
-Grow lots of potatoes
+Grow potatoes
 
 Narrative:
 In order to let my country eat chips
 As a farmer
-I want to grow potatoes
+I want to grow potatoes

serenity-model/src/main/java/net/thucydides/model/reports/html/ReportNameProvider.java

@@ -127,7 +127,7 @@ private String prefixUsing(Optional<String> context) {
     }
 
     public String forRequirement(Requirement requirement) {
-        return reportNamer.getNormalizedReportNameFor(prefixUsing(context) + "requirement_" + requirement.getPath());
+        return reportNamer.getNormalizedReportNameFor(prefixUsing(context) + requirement.getType() + "_" + requirement.getPath());
     }
 
     public String forRequirement(TestTag tag) {

serenity-model/src/main/java/net/thucydides/model/requirements/AggregateRequirements.java

@@ -1,14 +1,14 @@
 package net.thucydides.model.requirements;
 
-import net.serenitybdd.model.di.ModelInfrastructure;
-import net.serenitybdd.model.environment.ConfiguredEnvironment;
 import net.thucydides.model.domain.ReportType;
+import net.thucydides.model.environment.SystemEnvironmentVariables;
+import net.thucydides.model.issues.SystemPropertiesIssueTracking;
 import net.thucydides.model.reports.html.ReportNameProvider;
 import net.thucydides.model.requirements.reports.FileSystemRequirmentsOutcomeFactory;
 import net.thucydides.model.requirements.reports.RequirementsOutcomeFactory;
+import net.thucydides.model.util.EnvironmentVariables;
 
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.List;
 
 import static net.thucydides.model.reports.html.ReportNameProvider.NO_CONTEXT;
@@ -18,31 +18,36 @@
  */
 public class AggregateRequirements implements Requirements {
 
-    private final RequirementsService requirementsService;
-    private final RequirementsOutcomeFactory requirmentsOutcomeFactory;
+    private final BaseRequirementsService requirementsService;
+    private final RequirementsOutcomeFactory requirementsOutcomeFactory;
 
     public AggregateRequirements(Path jsonOutcomes, String featureFilesDirectory) {
+        this(jsonOutcomes, featureFilesDirectory, SystemEnvironmentVariables.createEnvironmentVariables());
+    }
+
+    public AggregateRequirements(Path jsonOutcomes, String featureFilesDirectory, EnvironmentVariables environmentVariables) {
         this.requirementsService = new AggregateRequirementsService(
-                ModelInfrastructure.getEnvironmentVariables(),
-                new FileSystemRequirementsTagProvider(featureFilesDirectory),
-                new TestOutcomeRequirementsTagProvider().fromSourceDirectory(jsonOutcomes));
-        this.requirmentsOutcomeFactory = new FileSystemRequirmentsOutcomeFactory(
-                ConfiguredEnvironment.getEnvironmentVariables(),
-                ModelInfrastructure.getIssueTracking(),
-                new ReportNameProvider(NO_CONTEXT, ReportType.HTML, getRequirementsService()),
-                featureFilesDirectory);
+                environmentVariables,
+                new FileSystemRequirementsTagProvider(featureFilesDirectory, environmentVariables),
+                new TestOutcomeRequirementsTagProvider(environmentVariables).fromSourceDirectory(jsonOutcomes)
+        );
+        this.requirementsOutcomeFactory = new FileSystemRequirmentsOutcomeFactory(
+                environmentVariables,
+                new SystemPropertiesIssueTracking(environmentVariables),
+                new ReportNameProvider(NO_CONTEXT, ReportType.HTML, this.requirementsService),
+                this.requirementsService
+        );
     }
 
     public RequirementsOutcomeFactory getRequirementsOutcomeFactory() {
-        return requirmentsOutcomeFactory;
+        return requirementsOutcomeFactory;
     }
 
-    public RequirementsService getRequirementsService() {
+    public BaseRequirementsService getRequirementsService() {
         return requirementsService;
     }
 
     public List<String> getTypes() {
         return requirementsService.getRequirementTypes();
     }
-
 }

serenity-model/src/main/java/net/thucydides/model/requirements/AggregateRequirementsService.java

@@ -1,33 +1,24 @@
 package net.thucydides.model.requirements;
 
-import net.serenitybdd.model.collect.NewList;
-import net.serenitybdd.model.di.ModelInfrastructure;
-import net.serenitybdd.model.environment.EnvironmentSpecificConfiguration;
-import net.thucydides.model.ThucydidesSystemProperty;
 import net.thucydides.model.domain.TestOutcome;
-import net.thucydides.model.environment.SystemEnvironmentVariables;
 import net.thucydides.model.requirements.model.Requirement;
-import net.thucydides.model.statistics.service.AnnotationBasedTagProvider;
-import net.thucydides.model.statistics.service.FeatureStoryTagProvider;
 import net.thucydides.model.util.EnvironmentVariables;
 import org.apache.commons.lang3.time.StopWatch;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 public class AggregateRequirementsService extends BaseRequirementsService implements RequirementsService {
 
-    private List<RequirementsTagProvider> requirementsTagProviders;
+    private final List<RequirementsTagProvider> requirementsTagProviders;
 
     public AggregateRequirementsService(EnvironmentVariables environmentVariables,
                                         RequirementsTagProvider... tagProviders) {
         super(environmentVariables);
-        this.requirementsTagProviders = NewList.of(tagProviders);
+        this.requirementsTagProviders = List.of(tagProviders);
     }
 
-
     private static final Logger LOGGER = LoggerFactory.getLogger(AggregateRequirementsService.class);
 
     @Override
@@ -38,7 +29,7 @@ public List<Requirement> getRequirements() {
             requirements = getRequirementsTagProviders().stream()
                     .parallel()
                     .flatMap(RequirementsProvided::asStream)
-                    .collect(Collectors.toCollection(MergedRequirementList::new));
+                    .collect(RequirementsCollector.merging());
 
             stopWatch.split();
             LOGGER.debug("Requirements loaded in {}", stopWatch.formatSplitTime());

serenity-model/src/main/java/net/thucydides/model/requirements/DefaultCapabilityTypes.java

@@ -14,6 +14,8 @@
 import java.util.Optional;
 
 import static net.thucydides.model.ThucydidesSystemProperty.SERENITY_REQUIREMENT_TYPES;
+import static net.thucydides.model.requirements.SpecFileFilters.cucumberFeatureFiles;
+import static net.thucydides.model.requirements.SpecFileFilters.javascriptSpecFiles;
 
 public class DefaultCapabilityTypes {
 
@@ -22,7 +24,8 @@ public class DefaultCapabilityTypes {
     private List<String> defaultCapabilityTypes;
 
     private SearchForFilesOfType cucumberFileMatcher;
-    SearchForFilesOfType jbehaveFileMatcher;
+    private SearchForFilesOfType jbehaveFileMatcher;
+    private SearchForFilesOfType javascriptSpecMatcher;
 
     private Map<String, List<String>> requirementsCache = new HashMap<>();
     public static DefaultCapabilityTypes instance() {
@@ -34,6 +37,7 @@ public void clear() {
         defaultCapabilityTypes = null;
         jbehaveFileMatcher = null;
         cucumberFileMatcher = null;
+        javascriptSpecMatcher = null;
     }
 
     public List<String> getRequirementTypes(EnvironmentVariables environmentVariables, Optional<Path> root) {
@@ -54,7 +58,11 @@ public List<String> getDefaultCapabilityTypes(Optional<Path> root) {
             }
             else if (cucumberFilesExist(root)) {
                 defaultCapabilityTypes = cucumberCapabilityTypes(root);
-            } else {
+            }
+            else if(javaScriptSpecsExist(root)) {
+                defaultCapabilityTypes = javaScriptCapabilityHierararchy(root);
+            }
+            else {
                 defaultCapabilityTypes = DEFAULT_CAPABILITY_TYPES;
             }
         }
@@ -85,14 +93,26 @@ private List<String> cucumberCapabilityTypes(Optional<Path> root) {
         }
     }
 
+    private List<String> javaScriptCapabilityHierararchy(Optional<Path> root) {
+        int directoryHierarchyDepth = getJavaScriptSpecMatcher(root).get().getMaxDepth();
+        switch (directoryHierarchyDepth) {
+            case 0:
+                return NewList.of("feature");
+            case 1:
+                return NewList.of("capability", "feature");
+            default:
+                return NewList.of("theme", "capability", "feature");
+        }
+    }
+
     private Optional<SearchForFilesOfType> getJBehaveFileMatcher(Optional<Path> root) {
         if (jbehaveFileMatcher != null) {
             return Optional.of(jbehaveFileMatcher);
         }
         try {
 //            Optional<Path> root = RootDirectory.definedIn(environmentVariables).featuresOrStoriesRootDirectory();// findResourcePath(rootRequirementsDirectory + "/stories");
             if (root.isPresent()) {
-                jbehaveFileMatcher = new SearchForFilesOfType(root.get(), ".story");
+                jbehaveFileMatcher = new SearchForFilesOfType(root.get(), SpecFileFilters.jbehaveStoryFiles());
                 Files.walkFileTree(root.get(), jbehaveFileMatcher);
                 return Optional.of(jbehaveFileMatcher);
             }
@@ -102,6 +122,25 @@ private Optional<SearchForFilesOfType> getJBehaveFileMatcher(Optional<Path> root
         return Optional.empty();
     }
 
+    private Optional<SearchForFilesOfType> getJavaScriptSpecMatcher(Optional<Path> root) {
+        if (javascriptSpecMatcher != null) {
+            return Optional.of(javascriptSpecMatcher);
+        }
+
+        try {
+            if (! root.isPresent()) {
+                return Optional.empty();
+            }
+
+            javascriptSpecMatcher = new SearchForFilesOfType(root.get(), javascriptSpecFiles());
+            Files.walkFileTree(root.get(), javascriptSpecMatcher);
+            return Optional.of(javascriptSpecMatcher);
+        }
+        catch (IOException e) {
+            return Optional.empty();
+        }
+    }
+
     private boolean jbehaveFilesExist(Optional<Path> root) {
         return (getJBehaveFileMatcher(root).isPresent() && getJBehaveFileMatcher(root).get().hasMatchingFiles());
     }
@@ -110,6 +149,10 @@ private boolean cucumberFilesExist(Optional<Path> root) {
         return (getCucumberFileMatcher(root).isPresent() && getCucumberFileMatcher(root).get().hasMatchingFiles());
     }
 
+    private boolean javaScriptSpecsExist(Optional<Path> root) {
+        return (getJavaScriptSpecMatcher(root).isPresent() && getJavaScriptSpecMatcher(root).get().hasMatchingFiles());
+    }
+
     public int startLevelForADepthOf(Optional<Path> root, EnvironmentVariables environmentVariables, int requirementsDepth) {
         return Math.max(0, getRequirementTypes(environmentVariables, root).size() - requirementsDepth);
     }
@@ -121,7 +164,7 @@ private Optional<SearchForFilesOfType> getCucumberFileMatcher(Optional<Path> roo
         try {
 //            Optional<Path> root = RootDirectory.definedIn(environmentVariables).featuresOrStoriesRootDirectory();// findResourcePath(rootRequirementsDirectory + "/stories");
             if (root.isPresent()) {
-                cucumberFileMatcher = new SearchForFilesOfType(root.get(), ".feature");
+                cucumberFileMatcher = new SearchForFilesOfType(root.get(), cucumberFeatureFiles());
                 Files.walkFileTree(root.get(), cucumberFileMatcher);
                 return Optional.of(cucumberFileMatcher);
             }