Skip to content

Commit

Permalink
Integrate Jim's changes with local work
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher committed Dec 24, 2024
2 parents 443b7ec + cad82eb commit ecaafd1
Show file tree
Hide file tree
Showing 6 changed files with 218 additions and 124 deletions.
30 changes: 27 additions & 3 deletions .github/workflows/NightlyBMDB_CLI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ jobs:
needs: build
strategy:
matrix:
sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
# sets: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" ]
sets: [ "98", "99" ]
steps:
- name: Create Docker Image Dir
run: mkdir /tmp/docker
Expand Down Expand Up @@ -205,6 +206,7 @@ jobs:
# Append to master summary in github workspace
jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
mv temp.json ${{ github.workspace }}/total_exec_summary.json
echo $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) >> ${{ github.workspace }}/exec_summary.ndjson
echo "Done!"
else
echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
Expand Down Expand Up @@ -299,9 +301,28 @@ jobs:
- name: Combine .summary artifacts into one file
run: jq -s 'add | sort_by(.file_path)' $(find "$GITHUB_WORKSPACE/bmdb-results" -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json

- name: Combine all exec_summary.ndjson files (from each of the matrix jobs) into one exec_summary.ndjson file
run: echo $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.ndjson') > $GITHUB_WORKSPACE/exec_summary.ndjson

- name: Upload combined exec_summary.ndjson file
uses: actions/upload-artifact@v3
with:
path: ${{ github.workspace }}/exec_summary.ndjson
name: exec_summary.ndjson

- name: Combine .summary artifacts into one file
run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json

# run exec-report CLI command via docker to generate report using the exec_summary.ndjson file and the embedded test_cases.ndjson file
- name: Run Docker test-report command
run: docker run -v ${{ github.workspace }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') test-report -c SYSBIO_BIOMD --exec-summaries /root/exec_summary.ndjson --report-md /root/report.md

- name: Upload generated report.md
uses: actions/upload-artifact@v3
with:
name: report.md
path: ${{ github.workspace }}/report.md

- name: Upload complete, compiled summary
uses: actions/upload-artifact@v3
with:
Expand All @@ -320,5 +341,8 @@ jobs:
- name: Post results to slack part 2
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage

- name: Post results to slack part 3
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="To see detailed logs of the BMDB results, see - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Go see what happened!" https://slack.com/api/chat.postMessage
- name: Post results to slack part 3 (report.md from exec-report CLI command)
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/report.md)" https://slack.com/api/chat.postMessage

- name: Post results to slack part 4
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="To see detailed logs of the BMDB results, see - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Go see what happened!" https://slack.com/api/chat.postMessage
47 changes: 47 additions & 0 deletions vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.vcell.cli.testsupport;


import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

Expand All @@ -19,10 +20,13 @@ public OmexTestCaseChange(OmexTestCase original, OmexTestCase updated) {
this.original = original;
this.updated = updated;
}

}

public static class OmexTestStatistics {
public int testCaseCount;
public int testCaseFailCount;
public int testCasePassCount;
public int testCaseChangeCount;
public int unmatchedTestCaseCount;
public int unmatchedExecutionsCount;
Expand Down Expand Up @@ -86,6 +90,8 @@ public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSu
}
}
statistics.testCaseCount = testCases.size();
statistics.testCasePassCount = (int) testCases.stream().filter(tc -> tc.known_status == OmexTestCase.Status.PASS).count();
statistics.testCaseFailCount = (int) testCases.stream().filter(tc -> tc.known_status == OmexTestCase.Status.FAIL).count();
statistics.totalExecutions = execSummaries.size();
statistics.failedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.FAILED).count();
statistics.passedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.PASSED).count();
Expand Down Expand Up @@ -131,4 +137,45 @@ public String toYaml() {
}
}

public String toMarkdown() throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
StringBuilder sb = new StringBuilder();
sb.append("# Test Report\n");
int testCasesFailed = statistics.testCaseFailCount;
int testCasesPassed = statistics.testCasePassCount;
int testCasesUnknown = statistics.testCaseCount - testCasesFailed - testCasesPassed;
sb.append("## Historical Test Case Records: "+statistics.testCaseCount+" (KNOWN PASSES = "+statistics.testCasePassCount+", KNOWN FAILURES = "+statistics.testCaseFailCount+", STATUS NOT RECORDED = "+testCasesUnknown).append(")\n");
sb.append("## New Test Executions: "+statistics.totalExecutions+" (PASSES = "+statistics.passedExecutionsCount+", FAILURES = "+statistics.failedExecutionsCount).append(")\n");

if (statistics.unmatchedTestCaseCount > 0) {
sb.append("## Historical Test Case Records without matching Test Executions: ").append(statistics.unmatchedTestCaseCount).append("\n");
for (OmexTestCase testCase : unmatchedTestCases) {
sb.append(" - ").append(testCase.test_collection).append(" : ").append(testCase.file_path).append("\n");
}
} else {
sb.append("## All Historical Test Case Records have matching Test Executions\n");
}

if (statistics.unmatchedExecutionsCount > 0) {
sb.append("## New Test Executions without matching Historical Test Case Records: ").append(statistics.unmatchedExecutionsCount).append("\n");
for (OmexExecSummary execSummary : unmatchedExecSummaries) {
sb.append(" - ").append(execSummary.file_path).append("\n");
}
} else {
sb.append("## All New Test Executions have matching Historical Test Case Records\n");
}

if (!testCaseChanges.isEmpty()) {
sb.append("## New Test Executions which differ from Historical Test Case Records: ").append(statistics.testCaseChangeCount).append("\n");
for (OmexTestCaseChange testCaseChange : testCaseChanges) {
String diff = "(" + testCaseChange.original.known_status + ":" + testCaseChange.original.known_failure_type + ") -> (" + testCaseChange.updated.known_status + ":" + testCaseChange.updated.known_failure_type + ")";
sb.append(" - ").append(testCaseChange.original.file_path).append(": " + diff + " ===New==> ").append(objectMapper.writeValueAsString(testCaseChange.updated)).append("`\n");
}
} else {
sb.append("## No New Test Executions differ from Historical Test Case Records\n");
}

return sb.toString();
}

}
41 changes: 29 additions & 12 deletions vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,28 @@
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.function.Predicate;

@Command(name = "test-report", description = "create test reports for a test suite")
public class TestReportCommand implements Callable<Integer> {

private final static Logger logger = LogManager.getLogger(TestReportCommand.class);

@Option(names = { "-t", "--test-cases" }, description = "[optional] test cases file - defaults to embedded test cases")
@Option(names = { "-t", "--test-cases" }, required = false, description = "[optional] test cases file - defaults to embedded test cases")
private File testCasesNdjsonFile;

@Option(names = { "-e", "--exec-summaries" }, required = true, description = "test results file e.g. exec_summary.ndjson")
private File execSummaryNdjsonFile;

@Option(names = { "-r", "--report" }, required = false, description = "filename for generated test report (e.g. ./report.md)")
private File reportFile;
// list of OmexTestingDatabase.TestCollection objects to include
@Option(names = { "-c", "--collections" }, required = true, description = "list of test collections to include")
private List<OmexTestingDatabase.TestCollection> collections;

// flag weather output should be json or yaml
@Option(names = { "-j", "--json" }, description = "output report in json format")
private boolean bJson = false;
@Option(names = { "--report-md" }, required = true, description = "filename for generated markdown test report (e.g. ./report.md)")
private File reportFile_md;

@Option(names = { "--report-json" }, required = false, description = "filename for generated json test report (e.g. ./report.json)")
private File reportFile_json;

@Option(names = { "-d", "--debug" }, description = "enable debug logging")
private boolean bDebug = false;
Expand All @@ -42,15 +46,28 @@ public Integer call() {
config.updateLoggers();

try {
List<OmexTestCase> testCaseList = OmexTestingDatabase.loadOmexTestCases();
// read fully text file into a string from file 'execSummaryNdjsonFile'
// read test cases and filter by collections
Predicate<OmexTestCase> omexTestCasePredicate = tc -> collections.contains(tc.test_collection);
final List<OmexTestCase> testCaseList;
if (testCasesNdjsonFile != null) {
String test_cases_contents = Files.readString(testCasesNdjsonFile.toPath());
testCaseList = OmexTestingDatabase.parseOmexTestCases(test_cases_contents).stream().filter(omexTestCasePredicate).toList();
} else {
// if file not specified, use embedded test cases
testCaseList = OmexTestingDatabase.loadOmexTestCases().stream().filter(omexTestCasePredicate).toList();
}

// read exec summaries
String exec_summary_contents = Files.readString(execSummaryNdjsonFile.toPath());
List<OmexExecSummary> execSummaries = OmexTestingDatabase.loadOmexExecSummaries(exec_summary_contents);

// generate report
OmexTestReport report = OmexTestingDatabase.generateReport(testCaseList, execSummaries);
if (reportFile != null) {
Files.writeString(reportFile.toPath(), bJson ? report.toJson() : report.toYaml());
} else {
System.out.println(bJson ? report.toJson() : report.toYaml());
if (reportFile_json != null) {
Files.writeString(reportFile_json.toPath(), report.toJson());
}
if (reportFile_md != null) {
Files.writeString(reportFile_md.toPath(), report.toMarkdown());
}
return 0;
} catch (Exception e) {
Expand Down
Loading

0 comments on commit ecaafd1

Please sign in to comment.