Skip to content

Commit

Permalink
Fine grain test inclusion/exclusion (#104)
Browse files Browse the repository at this point in the history
- Adds new always aborting evaluator
- Adds override keyword to all evaluators (to allow naming and referencing evaluators)
- Adds additional properties to allow evaluator based abortion or suppression
- Adds override keyword to the Flight Evaluation Report
- Adds new properties to the Readme

Resolves #102
{minor}

Signed-off-by: Esta Nagy <[email protected]>
  • Loading branch information
nagyesta authored Jun 12, 2022
1 parent b2490f9 commit ac065b7
Show file tree
Hide file tree
Showing 21 changed files with 888 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ protected Map<String, Consumer<AbortMissionCommandOps>> defineOutline() {

private final Map<String, StageTimeStopwatch> stopwatchMap = new HashMap<>();

/**
* Returns a {@link MissionHealthCheckMatcher} matching any scenario URI.
*
* @return matcher
*/
public static MissionHealthCheckMatcher anyScenarioMatcher() {
return new ScenarioUriMatcher(".*");
}

/**
* Returns a {@link MissionHealthCheckMatcher} matching scenario URIs.
*
Expand Down
14 changes: 8 additions & 6 deletions mission-control/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,18 @@ In order to configure Abort-Mission, the easiest option would be to follow these

1. Implement [MissionOutline](./src/main/java/com/github/nagyesta/abortmission/core/outline/MissionOutline.java) named as `MissionOutlineDefinition`
preferably in your root package
2. Use tha annotations we provide [here](./src/main/java/com/github/nagyesta/abortmission/core/annotation/)
2. Use the annotations we provide [here](./src/main/java/com/github/nagyesta/abortmission/core/annotation/)
3. Make sure to hook our lifecycle methods by either
1. using a [LaunchSequenceTemplate](./src/main/java/com/github/nagyesta/abortmission/core/LaunchSequenceTemplate.java)
2. or one of the [Callable/runnable implementations here](./src/main/java/com/github/nagyesta/abortmission/core/selfpropelled/)
4. Figure out how to group your tests and limit the blast radius of each dependency as you go

### System properties

| Property | Type | Meaning |
| -------------------------------- | --------- | --------------------------------------------------------------------------- |
| `abort-mission.disarm.countdown` | `boolean` | Disables countdown aborts for all tests. Default: false |
| `abort-mission.disarm.mission` | `boolean` | Disables mission aborts for all tests. Default: false |
| `abort-mission.report.directory` | `String` | Output directory path where we want to save telemetry output. Default: null |
| Property | Type | Meaning |
|-------------------------------------------|-----------|------------------------------------------------------------------------------------------------------|
| `abort-mission.disarm.countdown` | `boolean` | Disables countdown aborts for all tests. Default: false |
| `abort-mission.disarm.mission` | `boolean` | Disables mission aborts for all tests. Default: false |
| `abort-mission.report.directory` | `String` | Output directory path where we want to save telemetry output. Default: null |
| `abort-mission.force.abort.evaluators` | `String` | Comma separated list of key words identifying evaluators which need to always abort. Default: null |
| `abort-mission.suppress.abort.evaluators` | `String` | Comma separated list of key words identifying evaluators which need to suppress abort. Default: null |
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,10 @@ protected Optional<StageTimeStopwatch> performPreLaunchInit(final Class<?> testI

final StageTimeStopwatch watch = new StageTimeStopwatch(testInstanceClass);
final Set<MissionHealthCheckEvaluator> evaluators = classBasedEvaluatorLookup.apply(testInstanceClass);
final boolean hasSuppression = evaluators.stream().anyMatch(MissionHealthCheckEvaluator::shouldSuppressAbort);
final boolean reportingDone = evaluateAndAbortIfNeeded(
partitionBy(evaluators, MissionHealthCheckEvaluator::shouldAbortCountdown),
partitionBy(evaluators, missionHealthCheckEvaluator
-> !hasSuppression && missionHealthCheckEvaluator.shouldAbortCountdown()),
annotationContextEvaluator().isAbortSuppressed(testInstanceClass),
watch.stop(),
MissionHealthCheckEvaluator::countdownLogger);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ public AbstractMissionLaunchSequenceTemplate(final Runnable abortSequence) {
protected Optional<StageTimeStopwatch> evaluateLaunchAbort(final Set<MissionHealthCheckEvaluator> evaluators,
final StageTimeStopwatch stopwatch,
final Supplier<Boolean> abortSuppressionDecisionSupplier) {
final Boolean shouldSuppressAbortDecisions = Objects.requireNonNull(abortSuppressionDecisionSupplier).get();
final boolean hasEvaluatorThatSuppressesAbort = evaluators.stream().anyMatch(MissionHealthCheckEvaluator::shouldSuppressAbort);
final boolean shouldSuppressAbortDecisions = Boolean.TRUE.equals(Objects.requireNonNull(abortSuppressionDecisionSupplier).get());
final boolean reportingDone = evaluateAndAbortIfNeeded(
partitionBy(evaluators, MissionHealthCheckEvaluator::shouldAbort),
Boolean.TRUE.equals(shouldSuppressAbortDecisions),
partitionBy(evaluators, missionHealthCheckEvaluator
-> !hasEvaluatorThatSuppressesAbort && missionHealthCheckEvaluator.shouldAbort()),
shouldSuppressAbortDecisions,
stopwatch.stop(),
MissionHealthCheckEvaluator::missionLogger);
return emptyIfTrue(reportingDone, stopwatch);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@
import com.github.nagyesta.abortmission.core.annotation.AnnotationContextEvaluator;
import com.github.nagyesta.abortmission.core.healthcheck.MissionHealthCheckEvaluator;
import com.github.nagyesta.abortmission.core.healthcheck.StageStatisticsCollectorFactory;
import com.github.nagyesta.abortmission.core.healthcheck.impl.DefaultStageStatisticsCollectorFactory;
import com.github.nagyesta.abortmission.core.healthcheck.impl.MissionStatisticsCollector;
import com.github.nagyesta.abortmission.core.healthcheck.impl.PercentageBasedMissionHealthCheckEvaluator;
import com.github.nagyesta.abortmission.core.healthcheck.impl.ReportOnlyMissionHealthCheckEvaluator;
import com.github.nagyesta.abortmission.core.healthcheck.impl.*;
import com.github.nagyesta.abortmission.core.matcher.MissionHealthCheckMatcher;
import com.github.nagyesta.abortmission.core.matcher.impl.MissionHealthCheckMatcherBuilder;
import com.github.nagyesta.abortmission.core.matcher.impl.builder.InitialMissionHealthCheckMatcherBuilder;
Expand All @@ -20,7 +17,14 @@
* Provides shorthands for the core functionality of the library.
*/
public final class MissionControl {

/**
* System property name we need to use to force aborts for the selected evaluators.
*/
public static final String ABORT_MISSION_FORCE_ABORT_EVALUATORS = "abort-mission.force.abort.evaluators";
/**
* System property name we need to use to suppress aborts for the selected evaluators.
*/
public static final String ABORT_MISSION_SUPPRESS_ABORT_EVALUATORS = "abort-mission.suppress.abort.evaluators";
/**
* System property name we need to use to disarm mission aborts.
*/
Expand Down Expand Up @@ -74,6 +78,32 @@ public static PercentageBasedMissionHealthCheckEvaluator.Builder percentageBased
statisticsFactory.newMissionStatistics(matcher)));
}

/**
* Creates a builder instance for always aborting evaluators.
*
* @param matcher The matcher we want to use for this evaluator.
* @return builder
*/
public static AlwaysAbortingMissionHealthCheckEvaluator.Builder abortingEvaluator(final MissionHealthCheckMatcher matcher) {
return abortingEvaluator(matcher, new DefaultStageStatisticsCollectorFactory());
}

/**
* Creates a builder instance for always aborting evaluators.
*
* @param matcher The matcher we want to use for this evaluator.
* @param statisticsFactory The factory instance that can provide statistics collectors.
* @return builder
*/
public static AlwaysAbortingMissionHealthCheckEvaluator.Builder abortingEvaluator(
final MissionHealthCheckMatcher matcher,
final StageStatisticsCollectorFactory statisticsFactory) {
return AlwaysAbortingMissionHealthCheckEvaluator
.builder(matcher, new MissionStatisticsCollector(
statisticsFactory.newCountdownStatistics(matcher),
statisticsFactory.newMissionStatistics(matcher)));
}

/**
* Creates a builder instance for report only evaluators.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,22 @@ public interface MissionHealthCheckEvaluator {
*/
ReadOnlyStageStatistics getMissionStatistics();

/**
* Tells us whether this evaluator should never abort and suppress other matching evaluators if they would.
*
* @return true if the evaluator should suppress abort, false otherwise.
*/
boolean shouldSuppressAbort();

/**
* Tells us whether this evaluator should always abort.
* <b>Important:</b> If two different evaluators are matching and one returns true for {@link #shouldSuppressAbort()} while
* the other returns true for {@code shouldForceAbort()}, then the "suppression" will take precedence and neither will abort.
*
* @return true if the evaluator should always abort.
*/
boolean shouldForceAbort();

/**
* Tells the launch controller whether we should abort the launch after the preparation was done.
*
Expand Down Expand Up @@ -70,4 +86,11 @@ public interface MissionHealthCheckEvaluator {
* @return mission
*/
StatisticsLogger missionLogger();

/**
* Returns the keyword we can use for overriding abort/disarm decisions for the evaluator using command line parameters.
*
* @return override keyword
*/
String overrideKeyword();
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@
import com.github.nagyesta.abortmission.core.healthcheck.StatisticsLogger;
import com.github.nagyesta.abortmission.core.matcher.MissionHealthCheckMatcher;

import static com.github.nagyesta.abortmission.core.MissionControl.ABORT_MISSION_DISARM_COUNTDOWN;
import static com.github.nagyesta.abortmission.core.MissionControl.ABORT_MISSION_DISARM_MISSION;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;

import static com.github.nagyesta.abortmission.core.MissionControl.*;

/**
* Implements the common functionality of {@link MissionHealthCheckEvaluator} instances.
Expand All @@ -16,16 +19,37 @@ public abstract class AbstractMissionHealthCheckEvaluator implements MissionHeal

private final MissionHealthCheckMatcher matcher;
private final MissionStatisticsCollector stats;
private final String overrideKeyword;

/**
* Sets the matcher and the mission statistics collector.
*
* @param matcher The health check matcher mentioned by {@link MissionHealthCheckEvaluator#getMatcher()}.
* @param stats The statistics collector mentioned by {@link MissionHealthCheckEvaluator#getStats()}.
*/
protected AbstractMissionHealthCheckEvaluator(final MissionHealthCheckMatcher matcher, final MissionStatisticsCollector stats) {
protected AbstractMissionHealthCheckEvaluator(final MissionHealthCheckMatcher matcher,
final MissionStatisticsCollector stats) {
this(matcher, stats, null);
}

/**
* Sets the matcher and the mission statistics collector.
*
* @param matcher The health check matcher mentioned by {@link MissionHealthCheckEvaluator#getMatcher()}.
* @param stats The statistics collector mentioned by {@link MissionHealthCheckEvaluator#getStats()}.
* @param overrideKeyword The keyword used for fine-grained abort/disarm overrides.
*/
protected AbstractMissionHealthCheckEvaluator(final MissionHealthCheckMatcher matcher,
final MissionStatisticsCollector stats,
final String overrideKeyword) {
this.matcher = matcher;
this.stats = stats;
this.overrideKeyword = overrideKeyword;
}

@Override
public String overrideKeyword() {
return overrideKeyword;
}

@Override
Expand Down Expand Up @@ -58,20 +82,45 @@ public StatisticsLogger missionLogger() {
return stats.getMission();
}

@Override
public boolean shouldSuppressAbort() {
return evaluateOverrideList(ABORT_MISSION_SUPPRESS_ABORT_EVALUATORS)
.contains(overrideKeyword);
}

@Override
public boolean shouldForceAbort() {
return evaluateOverrideList(ABORT_MISSION_FORCE_ABORT_EVALUATORS)
.contains(overrideKeyword);
}

@Override
public boolean shouldAbort() {
if (isDisarmed(ABORT_MISSION_DISARM_MISSION)) {
return false;
}
return shouldAbortInternal();
return shouldForceAbort() || shouldAbortInternal();
}

@Override
public boolean shouldAbortCountdown() {
if (isDisarmed(ABORT_MISSION_DISARM_COUNTDOWN)) {
return false;
}
return shouldAbortCountdownInternal();
return shouldForceAbort() || shouldAbortCountdownInternal();
}

/**
* Returns the set of evaluator keywords which are defined in the given System property.
*
* @param propertyName The name of the System property.
* @return The tokenized set of keywords defined by the property value or empty set if not defined.
*/
protected Set<String> evaluateOverrideList(final String propertyName) {
final String property = System.getProperty(propertyName, "");
return Arrays.stream(property.split(","))
.filter(s -> !s.isBlank())
.collect(Collectors.toSet());
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package com.github.nagyesta.abortmission.core.healthcheck.impl;

import com.github.nagyesta.abortmission.core.matcher.MissionHealthCheckMatcher;

import java.util.Objects;

/**
* {@link com.github.nagyesta.abortmission.core.healthcheck.MissionHealthCheckEvaluator} implementation intended to always abort.
*/
@SuppressWarnings("checkstyle:FinalClass")
public class AlwaysAbortingMissionHealthCheckEvaluator extends AbstractMissionHealthCheckEvaluator {

private AlwaysAbortingMissionHealthCheckEvaluator(final Builder builder) {
super(Objects.requireNonNull(builder, "Builder cannot be null.").matcher,
builder.statisticsCollector, builder.overrideKeyword);
}

public static Builder builder(final MissionHealthCheckMatcher matcher,
final MissionStatisticsCollector statisticsCollector) {
return new Builder(matcher, statisticsCollector);
}

@Override
public int getBurnInTestCount() {
return Integer.MAX_VALUE;
}

@Override
protected boolean shouldAbortInternal() {
return true;
}

@Override
protected boolean shouldAbortCountdownInternal() {
return true;
}

@SuppressWarnings({"checkstyle:HiddenField", "checkstyle:DesignForExtension"})
public static final class Builder {
private final MissionHealthCheckMatcher matcher;
private final MissionStatisticsCollector statisticsCollector;
private String overrideKeyword;

private Builder(final MissionHealthCheckMatcher matcher,
final MissionStatisticsCollector statisticsCollector) {
this.matcher = Objects.requireNonNull(matcher, "Matcher cannot be null.");
this.statisticsCollector = Objects.requireNonNull(statisticsCollector, "Statistic collector cannot be null.");
}

public Builder overrideKeyword(final String overrideKeyword) {
if (overrideKeyword == null || overrideKeyword.isBlank()) {
throw new IllegalArgumentException("Override keyword must br non-blank.");
} else if (!overrideKeyword.matches("[\\da-zA-Z\\-]+")) {
throw new IllegalArgumentException("Override keyword must contain only alpha-numeric characters and dash (a-zA-Z0-9\\-).");
}
this.overrideKeyword = overrideKeyword;
return this;
}

public AlwaysAbortingMissionHealthCheckEvaluator build() {
return new AlwaysAbortingMissionHealthCheckEvaluator(this);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public class PercentageBasedMissionHealthCheckEvaluator extends AbstractMissionH

private PercentageBasedMissionHealthCheckEvaluator(final Builder builder) {
super(Objects.requireNonNull(builder, "Builder cannot be null.").matcher,
builder.statisticsCollector);
builder.statisticsCollector, builder.overrideKeyword);
this.burnInTestCount = builder.burnInTestCount;
this.abortThreshold = builder.abortThreshold;
}
Expand Down Expand Up @@ -65,6 +65,7 @@ public static final class Builder {
private final MissionHealthCheckMatcher matcher;
private int burnInTestCount = BURN_IN_LOWER_LIMIT;
private int abortThreshold = PERCENTAGE_LOWER_LIMIT;
private String overrideKeyword;

private Builder(final MissionHealthCheckMatcher matcher, final MissionStatisticsCollector statisticsCollector) {
this.matcher = Objects.requireNonNull(matcher, "Matcher cannot be null.");
Expand All @@ -87,6 +88,16 @@ public Builder abortThreshold(final int abortThreshold) {
return this;
}

public Builder overrideKeyword(final String overrideKeyword) {
if (overrideKeyword == null || overrideKeyword.isBlank()) {
throw new IllegalArgumentException("Override keyword must br non-blank.");
} else if (!overrideKeyword.matches("[\\da-zA-Z\\-]+")) {
throw new IllegalArgumentException("Override keyword must contain only alpha-numeric characters and dash (a-zA-Z0-9\\-).");
}
this.overrideKeyword = overrideKeyword;
return this;
}

public PercentageBasedMissionHealthCheckEvaluator build() {
return new PercentageBasedMissionHealthCheckEvaluator(this);
}
Expand Down
Loading

0 comments on commit ac065b7

Please sign in to comment.