Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Java tests refactoring #84

Open
wants to merge 3 commits into
base: refactor-pipes-tests
Choose a base branch
from
Open

Java tests refactoring #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cc7b8f9
Big tests refactoring
GingerYouth Dec 25, 2024
0972036
Tests refactoring
GingerYouth Dec 28, 2024
a6d51da
remove unused import
GingerYouth Dec 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ...ries/pipes/implementations/java/dagster-pipes-java/src/main/java/pipes/utils/Example.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
import pipes.DagsterPipesException;
import pipes.PipesContext;
import pipes.PipesSession;
import pipes.data.PipesMetadata;
import pipes.loaders.PipesContextLoader;
import pipes.loaders.PipesDefaultContextLoader;
import pipes.loaders.PipesEnvVarParamsLoader;
import pipes.loaders.PipesParamsLoader;
import pipes.writers.PipesDefaultMessageWriter;
import pipes.writers.PipesMessageWriter;
import pipes.writers.PipesMessageWriterChannel;
import types.Type;

import java.util.HashMap;
import java.util.Map;
Expand Down
107 changes: 25 additions & 82 deletions libraries/pipes/implementations/java/dagster-pipes-java/src/test/java/pipes/MainTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import picocli.CommandLine;
import pipes.data.PipesAssetCheckSeverity;
import pipes.data.PipesConstants;
import pipes.loaders.PipesContextLoader;
import pipes.loaders.PipesDefaultContextLoader;
import pipes.loaders.PipesS3ContextLoader;
Expand All @@ -16,7 +15,6 @@

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

Expand All @@ -29,25 +27,6 @@ public class MainTest implements Runnable {
private final ObjectMapper objectMapper = new ObjectMapper();
private Map<String, Object> cachedJson = new ConcurrentHashMap<>();

@CommandLine.Option(
names = {"--context"},
description = "Provide DAGSTER_PIPES_CONTEXT value for testing"
)
private String context;

@CommandLine.Option(
names = {"--messages"},
description = "Provide DAGSTER_PIPES_MESSAGES value for testing"
)
private String messages;

@CommandLine.Option(
names = {"--env"},
description = "Get DAGSTER_PIPES_MESSAGES & DAGSTER_PIPES_CONTEXT values " +
"from environmental variables"
)
private boolean env = false;

@CommandLine.Option(
names = {"--job-name"},
description = "Provide value of 'jobName' for testing"
Expand All @@ -61,13 +40,7 @@ public class MainTest implements Runnable {
private String extras;

@CommandLine.Option(
names = {"--full"},
description = "Flag to test full PipesContext usage"
)
private boolean full = false;

@CommandLine.Option(
names = {"--custom-payload-path"},
names = {"--custom-payload"},
description = "Specify custom payload path"
)
private String customPayloadPath;
Expand All @@ -84,18 +57,6 @@ public class MainTest implements Runnable {
)
private String reportAssetMaterializationJson;

@CommandLine.Option(
names = {"--throw-error"},
description = "Throw exception in PipesSession with specified message"
)
private boolean throwException = false;

@CommandLine.Option(
names = {"--logging"},
description = "Flag to test logging"
)
private boolean logging = false;

@CommandLine.Option(
names = {"--message-writer"},
description = "Specify the type of the message writer: default,s3"
Expand All @@ -108,47 +69,34 @@ public class MainTest implements Runnable {
)
private String contextLoaderType;

@CommandLine.Option(
names = {"--test-name"},
description = "Specify the name of the test"
)
private String testName;

@Override
public void run() {
Map<String, String> input = new HashMap<>();
PipesTests pipesTests = new PipesTests();
try {
if (this.context != null) {
input.put(PipesConstants.CONTEXT_ENV_VAR.name, context);
}
if (this.messages != null) {
input.put(PipesConstants.MESSAGES_ENV_VAR.name, this.messages);
}
pipesTests.setInput(input);

final PipesContextLoader loader;
if (this.contextLoaderType != null && !this.contextLoaderType.isEmpty()) {
switch (this.contextLoaderType) {
case "s3":
S3Client amazonS3Client = S3Client.builder().build();
loader = new PipesS3ContextLoader(amazonS3Client);
break;
case "default":
loader = new PipesDefaultContextLoader();
break;
default:
throw new IllegalArgumentException("Specified unknown context loader type!");
if (this.contextLoaderType.equals("s3")) {
S3Client amazonS3Client = S3Client.builder().build();
loader = new PipesS3ContextLoader(amazonS3Client);
} else {
loader = new PipesDefaultContextLoader();
}
pipesTests.setContextLoader(loader);
}

final PipesMessageWriter<? extends PipesMessageWriterChannel> writer;
if (this.messageWriter != null && !this.messageWriter.isEmpty()) {
switch (this.messageWriter) {
case "s3":
S3Client amazonS3Client = S3Client.builder().build();
writer = new PipesS3MessageWriter(amazonS3Client);
break;
case "default":
writer = new PipesDefaultMessageWriter();
break;
default:
throw new IllegalArgumentException("Specified unknown message writer!");
if (this.messageWriter.equals("s3")) {
S3Client amazonS3Client = S3Client.builder().build();
writer = new PipesS3MessageWriter(amazonS3Client);
} else {
writer = new PipesDefaultMessageWriter();
}
pipesTests.setMessageWriter(writer);
}
Expand All @@ -160,12 +108,12 @@ public void run() {
pipesTests.setPayload(payload);
}

if (this.throwException) {
if (this.testName != null && this.testName.equals("test_error_reporting")) {
pipesTests.testRunPipesSessionWithException();
return;
}

if (this.logging) {
if (this.testName != null && this.testName.equals("test_message_log")) {
pipesTests.testLogging();
return;
}
Expand All @@ -184,31 +132,26 @@ public void run() {
String checkName = loadParamByWrapperKey("checkName", String.class);
boolean passed = loadParamByWrapperKey("passed", Boolean.class);
String assetKey = loadParamByWrapperKey("assetKey", String.class);
PipesAssetCheckSeverity severity = PipesAssetCheckSeverity.valueOf(loadParamByWrapperKey("severity", String.class));
PipesAssetCheckSeverity severity = PipesAssetCheckSeverity.valueOf(
loadParamByWrapperKey("severity", String.class)
);
pipesTests.setCheck(checkName, passed, assetKey, severity);
}

if (this.full) {
pipesTests.fullTest();
return;
} else {
pipesTests.setContextData();
}

if (this.extras != null) {
File jsonFile = new File(this.extras);
ObjectMapper objectMapper = new ObjectMapper();
Map<String, Object> extrasMap = objectMapper.readValue(
jsonFile, new TypeReference<Map<String, Object>>() {}
jsonFile, new TypeReference<Map<String,Object>>() {}
);
pipesTests.setExtras(extrasMap);
pipesTests.testExtras();
}

if (this.jobName != null) {
pipesTests.setJobName(this.jobName);
pipesTests.testJobName();
}

pipesTests.fullTest();
} catch (IOException | DagsterPipesException exception) {
throw new RuntimeException(exception);
}
Expand Down
47 changes: 17 additions & 30 deletions libraries/pipes/implementations/java/dagster-pipes-java/src/test/java/pipes/PipesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
@Disabled
public class PipesTests {

private Map<String, String> input;
private PipesContextLoader contextLoader;
private PipesContextData contextData;
private Map<String, Object> extras;
private String jobName;
private Object payload;
Expand All @@ -41,10 +39,6 @@ public class PipesTests {
//Message writer
private PipesMessageWriter<? extends PipesMessageWriterChannel> pipesMessageWriter;

void setInput(Map<String, String> input) {
this.input = input;
}

void setExtras(Map<String, Object> extras) {
this.extras = extras;
}
Expand All @@ -53,15 +47,10 @@ void setJobName(String jobName) {
this.jobName = jobName;
}

void setContextData() throws DagsterPipesException {
this.contextData = DataLoader.getData(input);
}

void setContextLoader(PipesContextLoader contextLoader) throws DagsterPipesException {
this.contextLoader = contextLoader;
}


void setPayload(Object payload) {
this.payload = payload;
}
Expand All @@ -84,25 +73,6 @@ void setMessageWriter(PipesMessageWriter<? extends PipesMessageWriterChannel> wr
this.pipesMessageWriter = writer;
}

@Test
void testExtras() {
Assertions.assertTrue(
contextData.getExtras().entrySet().containsAll(this.extras.entrySet()),
"Extras does not contain all provided entries."
);
System.out.println("Extras are correct.");
}

@Test
void testJobName() {
Assertions.assertEquals(
this.jobName,
contextData.getJobName(),
"JobName is incorrect."
);
System.out.println("JobName is correct.");
}

@Test
void fullTest() throws DagsterPipesException {
getTestSession().runDagsterPipes(this::fullTest);
Expand All @@ -111,6 +81,23 @@ void fullTest() throws DagsterPipesException {
private void fullTest(PipesContext context) throws DagsterPipesException {
context.reportCustomMessage("Hello from external process!");

if (this.extras != null) {
Assertions.assertTrue(
context.getExtras().entrySet().containsAll(this.extras.entrySet()),
"Extras does not contain all provided entries."
);
System.out.println("Extras are correct.");
}

if (this.jobName != null) {
Assertions.assertEquals(
this.jobName,
context.getJobName(),
"JobName is incorrect."
);
System.out.println("JobName is correct.");
}

if (this.payload != null) {
context.reportCustomMessage(this.payload);
System.out.println("Payload reported with custom message.");
Expand Down
Loading