Skip to content

Commit

Permalink
Fixed #188 treating errors and exceptions in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@baubakg
baubakg committed Aug 19, 2024
1 parent 2537a22 commit 9f8b555
Show file tree
Hide file tree
Showing 4 changed files with 152 additions and 29 deletions.
22 changes: 11 additions & 11 deletions src/main/java/com/adobe/campaign/tests/integro/phased/Mutational.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,27 @@
@Test(dataProvider = "MUTATIONAL", dataProviderClass = PhasedDataProvider.class)
public abstract class Mutational {

public void shuffled(String phaseGroup) {
public void shuffled(String phaseGroup) throws Throwable {

String l_thisScneario = this.getClass().getTypeName()+"("+phaseGroup+")";
String l_thisScneario = this.getClass().getTypeName() + "(" + phaseGroup + ")";

PhasedTestManager.ScenarioContextData x = PhasedTestManager.getScenarioContext().get(l_thisScneario);

Class l_executingClass = this.getClass();

Map<String, ScenarioStepDependencies> l_scenarioDependencies = PhasedTestManager.getStepDependencies();


List<StepDependencies> l_orderList = Phases.getCurrentPhase()
.equals(Phases.PERMUTATIONAL) ? l_scenarioDependencies.get(l_executingClass.getTypeName())
.fetchScenarioPermutations().get(phaseGroup) : l_scenarioDependencies.get(
l_executingClass.getTypeName()).fetchExecutionOrderList();

// var nrOfSteps = Phases.getCurrentPhase().hasSplittingEvent() ? PhasedTestManager.fetchShuffledStepCount(
// phaseGroup)[0] : l_orderList.size();
// var nrOfSteps = Phases.getCurrentPhase().hasSplittingEvent() ? PhasedTestManager.fetchShuffledStepCount(
// phaseGroup)[0] : l_orderList.size();

Integer[] l_boundaries = MutationManager.fetchExecutionIndex(l_executingClass.getTypeName(), phaseGroup, Phases.getCurrentPhase() );
// System.out.println(nrOfSteps + " - " + phaseGroup);
Integer[] l_boundaries = MutationManager.fetchExecutionIndex(l_executingClass.getTypeName(), phaseGroup,
Phases.getCurrentPhase());
// System.out.println(nrOfSteps + " - " + phaseGroup);
//for (Method stepMethod : l_executingClass.getDeclaredMethods()) {
//for (StepDependencies stepOrdering : stepOrder) {

Expand All @@ -53,7 +53,7 @@ public void shuffled(String phaseGroup) {
//String lt_currentStepName = stepOrder.get(i).getStepName();
//Method stepMethod = Arrays.stream(l_executingClass.getMethods()).filter(m -> m.getName().equals(lt_currentStepName)).findFirst().get();
String stepName = l_orderList.get(i).getStepName();
String stepId = l_executingClass.getTypeName() + "." + stepName+"("+phaseGroup+")";
String stepId = l_executingClass.getTypeName() + "." + stepName + "(" + phaseGroup + ")";

Method stepMethod = Arrays.stream(l_executingClass.getDeclaredMethods())
.filter(dm -> dm.getName().equals(stepName)).findFirst().get();
Expand All @@ -75,7 +75,6 @@ public void shuffled(String phaseGroup) {
stepMethod.invoke(ourInstance, phaseGroup);
long l_end = System.currentTimeMillis();


if (Phases.ASYNCHRONOUS.isSelected()) {
//Check if there is an event declared
String lt_event = PhasedEventManager.fetchEvent(stepMethod, phaseGroup);
Expand All @@ -86,12 +85,13 @@ public void shuffled(String phaseGroup) {
}

PhasedTestManager.scenarioStateStore(PhasedTestManager.fetchScenarioName(stepMethod, phaseGroup),
ClassPathParser.fetchFullName(stepMethod), TestResult.SUCCESS,l_start,l_end);
ClassPathParser.fetchFullName(stepMethod), TestResult.SUCCESS, l_start, l_end);

} catch (IllegalAccessException e) {
throw new RuntimeException(e);
} catch (InvocationTargetException e) {
throw new RuntimeException(e);
Throwable targetException = e.getTargetException();
throw targetException;
} catch (InstantiationException e) {
throw new RuntimeException(e);
} catch (NoSuchMethodException e) {
Expand Down
120 changes: 104 additions & 16 deletions src/test/java/com/adobe/campaign/tests/integro/phased/MutationalTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,9 @@
*/
package com.adobe.campaign.tests.integro.phased;

import com.adobe.campaign.tests.integro.phased.ConfigValueHandlerPhased;
import com.adobe.campaign.tests.integro.phased.MutationListener;
import com.adobe.campaign.tests.integro.phased.PhasedTestManager;
import com.adobe.campaign.tests.integro.phased.Phases;
import com.adobe.campaign.tests.integro.phased.data.events.MyNonInterruptiveEvent;
import com.adobe.campaign.tests.integro.phased.mutational.data.erroneous.IE_Shuffled_ErrorAssertion1;
import com.adobe.campaign.tests.integro.phased.mutational.data.erroneous.IE_Shuffled_ErrorOtherNonAssertive1;
import com.adobe.campaign.tests.integro.phased.mutational.data.nie.TestMutationalShuffled_eventPassedAsExecutionVariable;
import com.adobe.campaign.tests.integro.phased.mutational.data.permutational.MultipleProducerConsumer;
import com.adobe.campaign.tests.integro.phased.mutational.data.permutational.ShoppingCartDemo;
Expand All @@ -25,7 +23,6 @@
import org.testng.TestNG;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.testng.xml.XmlClass;
import org.testng.xml.XmlPackage;
Expand Down Expand Up @@ -58,7 +55,6 @@ public void resetVariables() {

PhasedTestManager.MergedReportData.resetReport();


//Delete standard cache file
File l_importCacheFile = new File(
GeneralTestUtils.fetchCacheDirectory(PhasedTestManager.STD_STORE_DIR),
Expand Down Expand Up @@ -119,6 +115,96 @@ public void testDefault() {
Matchers.equalTo(1));
}

@Test
public void testDefaultWithAssertionError() {

// Rampup
TestNG myTestNG = TestTools.createTestNG();
TestListenerAdapter tla = TestTools.fetchTestResultsHandler(myTestNG);

// Define suites
XmlSuite mySuite = TestTools.addSuitToTestNGTest(myTestNG, "Automated Suite Phased Testing");

// Add listeners
//mySuiteC.addListener(EventInjectorListener.class.getTypeName());
mySuite.addListener(MutationListener.class.getTypeName());

// Create an instance of XmlTest and assign a name for it.
XmlTest myTest = TestTools.attachTestToSuite(mySuite, "Test Repetetive Phased Tests Producer");

var l_testClass = IE_Shuffled_ErrorAssertion1.class;
myTest.setXmlClasses(Collections.singletonList(new XmlClass(l_testClass)));

// Add package to test

myTestNG.run();

assertThat("We should have no successful methods of phased Tests",
(int) tla.getPassedTests().size(),
is(equalTo(0)));

assertThat("We should have 1 failed method of phased Tests",
(int) tla.getFailedTests().size(),
is(equalTo(1)));

assertThat("We should have no skipped methods of phased Tests",
(int) tla.getSkippedTests().size(),
is(equalTo(0)));

assertThat("We should have no executions for the phased group 0",
tla.getFailedTests().stream().filter(m -> m.getInstanceName().equals(l_testClass.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(1));

assertThat("The exception should be an assertion exception", tla.getFailedTests().get(0).getThrowable(),
Matchers.instanceOf(AssertionError.class));

}

@Test
public void testDefaultWithException() {

// Rampup
TestNG myTestNG = TestTools.createTestNG();
TestListenerAdapter tla = TestTools.fetchTestResultsHandler(myTestNG);

// Define suites
XmlSuite mySuite = TestTools.addSuitToTestNGTest(myTestNG, "Automated Suite Phased Testing");

// Add listeners
mySuite.addListener(MutationListener.class.getTypeName());

// Create an instance of XmlTest and assign a name for it.
XmlTest myTest = TestTools.attachTestToSuite(mySuite, "Test Repetetive Phased Tests Producer");

var l_testClass = IE_Shuffled_ErrorOtherNonAssertive1.class;
myTest.setXmlClasses(Collections.singletonList(new XmlClass(l_testClass)));

// Add package to test

myTestNG.run();

assertThat("We should have no successful methods of phased Tests",
(int) tla.getPassedTests().size(),
is(equalTo(0)));

assertThat("We should have 1 failed method of phased Tests",
(int) tla.getFailedTests().size(),
is(equalTo(1)));

assertThat("We should have no skipped methods of phased Tests",
(int) tla.getSkippedTests().size(),
is(equalTo(0)));

assertThat("We should have no executions for the phased group 0",
tla.getFailedTests().stream().filter(m -> m.getInstanceName().equals(l_testClass.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(1));

assertThat("The exception should be an assertion exception", tla.getFailedTests().get(0).getThrowable(),
Matchers.instanceOf(IllegalArgumentException.class));

}

@Test
public void testInterruptiveEvent() {
Expand Down Expand Up @@ -156,11 +242,13 @@ public void testInterruptiveEvent() {
is(equalTo(5)));

assertThat("We should have no executions for the phased group 0",
tla.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild1.class.getTypeName())).collect(Collectors.toList()).size(),
tla.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild1.class.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(2));

assertThat("We should have no executions for the phased group 0",
tla.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild2.class.getTypeName())).collect(Collectors.toList()).size(),
tla.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild2.class.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(3));

//Add with consumer
Expand Down Expand Up @@ -190,22 +278,23 @@ public void testInterruptiveEvent() {

myTestNGC.run();

Map<String, PhasedTestManager.ScenarioContextData> x =PhasedTestManager.getScenarioContext();
Map<String, PhasedTestManager.ScenarioContextData> x = PhasedTestManager.getScenarioContext();

assertThat("We should have 2 successful method of phased Tests",
(int) tlaC.getPassedTests().size(),
is(equalTo(5)));

assertThat("We should have no executions for the phased group 0",
tlaC.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild1.class.getTypeName())).collect(Collectors.toList()).size(),
tlaC.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild1.class.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(2));

assertThat("We should have no executions for the phased group 0",
tlaC.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild2.class.getTypeName())).collect(Collectors.toList()).size(),
tlaC.getPassedTests().stream().filter(m -> m.getInstanceName().equals(PhasedChild2.class.getTypeName()))
.collect(Collectors.toList()).size(),
Matchers.equalTo(3));
}


@Test
public void testPermutational() {
//Activate Merge
Expand Down Expand Up @@ -237,7 +326,6 @@ public void testPermutational() {
(int) tla.getPassedTests().size(),
is(equalTo(2)));


}

@Test
Expand Down Expand Up @@ -272,7 +360,6 @@ public void testPermutationalDemo() {
is(equalTo(3)));
}


/**
* This is a test for non-intyerruptive events in shuffled classes
*/
Expand Down Expand Up @@ -301,7 +388,7 @@ public void testNonInterruptive_ParellelConfiguredAsExecutionVariable_Shuffled_O

myTestNG.run();

// assertThat("We should be in non-interruptive mode shuffled", PhasedTestManager.isPhasedTestShuffledMode(l_testClass));
// assertThat("We should be in non-interruptive mode shuffled", PhasedTestManager.isPhasedTestShuffledMode(l_testClass));

assertThat("We should have 3 successful executions of phased Tests",
(int) tla.getPassedTests().stream().filter(m -> m.getInstance().getClass().equals(l_testClass)).count(),
Expand All @@ -311,7 +398,8 @@ public void testNonInterruptive_ParellelConfiguredAsExecutionVariable_Shuffled_O
assertThat("We should have no failed tests", tla.getFailedTests().size(), equalTo(0));
assertThat("We should have no skipped tests", tla.getSkippedTests().size(), equalTo(0));

assertThat("We should have the correct number of events in the logs (1 x phase groups)", PhasedEventManager.getEventLogs().size(),
assertThat("We should have the correct number of events in the logs (1 x phase groups)",
PhasedEventManager.getEventLogs().size(),
Matchers.equalTo(6));
}
}
3 changes: 1 addition & 2 deletions ...al/data/erroneous/IE_SHUFFLED_ERROR1.java → ...rroneous/IE_Shuffled_ErrorAssertion1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
package com.adobe.campaign.tests.integro.phased.mutational.data.erroneous;

import com.adobe.campaign.tests.integro.phased.Mutational;
import com.adobe.campaign.tests.integro.phased.PhasedDataProvider;
import com.adobe.campaign.tests.integro.phased.PhasedTest;
import com.adobe.campaign.tests.integro.phased.PhasedTestManager;
import org.testng.annotations.Test;
Expand All @@ -19,7 +18,7 @@

@Test
@PhasedTest
public class IE_SHUFFLED_ERROR1 extends Mutational {
public class IE_Shuffled_ErrorAssertion1 extends Mutational {

public void step1(String val) {
System.out.println("step1 " + val);
Expand Down
36 changes: 36 additions & 0 deletions ...n/tests/integro/phased/mutational/data/erroneous/IE_Shuffled_ErrorOtherNonAssertive1.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright 2022 Adobe
* All Rights Reserved.
*
* NOTICE: Adobe permits you to use, modify, and distribute this file in
* accordance with the terms of the Adobe license agreement accompanying
* it.
*/
package com.adobe.campaign.tests.integro.phased.mutational.data.erroneous;

import com.adobe.campaign.tests.integro.phased.Mutational;
import com.adobe.campaign.tests.integro.phased.PhasedTest;
import com.adobe.campaign.tests.integro.phased.PhasedTestManager;
import org.testng.annotations.Test;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;

@Test
@PhasedTest
public class IE_Shuffled_ErrorOtherNonAssertive1 extends Mutational {

public void step1(String val) {
System.out.println("step1 " + val);
PhasedTestManager.produceInStep("A");
}

public void step2(String val) {
System.out.println("step2 " + val);
String l_fetchedValue = PhasedTestManager.consumeFromStep("step1");
throw new IllegalArgumentException("Bad Arghument");
}



}

0 comments on commit 9f8b555

Please sign in to comment.