diff --git a/vanilla-package/src/test/java/io/jenkins/jenkinsfile/runner/vanilla/SmokeTest.java b/vanilla-package/src/test/java/io/jenkins/jenkinsfile/runner/vanilla/SmokeTest.java
index aa66977..7228a94 100644
--- a/vanilla-package/src/test/java/io/jenkins/jenkinsfile/runner/vanilla/SmokeTest.java
+++ b/vanilla-package/src/test/java/io/jenkins/jenkinsfile/runner/vanilla/SmokeTest.java
@@ -109,7 +109,7 @@ public void shouldFailWithWrongJenkinsfile() throws Throwable {
 
         int result = new JFRTestUtil().run(jenkinsfile);
         assertThat("JFR should fail when there is no Jenkinsfile", result, not(equalTo(0)));
-        assertThat(systemOut.getLog(), containsString("No such file or directory"));
+        assertThat(systemOut.getLog(), containsString("FileNotFoundException"));
     }
 
     // TODO: uncomment once JFR can do something about timeouts internally