Add test automation

.github/workflows/build.yml

@@ -0,0 +1,29 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # See http://static.azul.com/zulu/bin/ for available JDK versions.
+        java: [8, 11, 14, 15-ea, 16-ea]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    name: Java ${{ matrix.java }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Set up java
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn -B verify

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test-src/test/resources/html5lib-tests"]
+	path = test-src/test/resources/html5lib-tests
+	url = [email protected]:html5lib/html5lib-tests.git

pom.xml

@@ -80,22 +80,13 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>1.5</source>
-          <target>1.5</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>
         <version>1.7</version>
-        <dependencies>
-          <dependency>
-            <groupId>com.sun</groupId>
-            <artifactId>tools</artifactId>
-            <version>1.5.0</version>
-            <scope>system</scope>
-            <systemPath>${java.home}/../lib/tools.jar</systemPath>
-          </dependency>
-        </dependencies>
         <executions>
           <execution>
             <id>intitialize-sources</id>
@@ -124,8 +115,12 @@
                 <property name="translator.sources" value="${basedir}/translator-src"/>
                 <property name="translator.classes" value="${project.build.directory}/translator-classes"/>
                 <mkdir dir="${translator.classes}"/>
-                <javac srcdir="${translator.sources}" includes="nu/validator/htmlparser/generator/ApplyHotSpotWorkaround.java" destdir="${translator.classes}" includeantruntime="false"/>
-                <java classname="nu.validator.htmlparser.generator.ApplyHotSpotWorkaround">
+                <javac srcdir="${translator.sources}" destdir="${translator.classes}"
+                       includes="nu/validator/htmlparser/generator/ApplyHotSpotWorkaround.java"
+                       includeantruntime="false"
+                       fork="${fork}"/>
+                <java classname="nu.validator.htmlparser.generator.ApplyHotSpotWorkaround"
+                      fork="${fork}">
                   <classpath>
                     <pathelement location="${translator.classes}"/>
                   </classpath>
@@ -141,7 +136,11 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <skip>true</skip>
+          <test>Html5libTest</test>
+          <testFailureIgnore>true</testFailureIgnore> <!-- FIXME: Remove this testFailureIgnore after we have all tests passing -->
+          <additionalClasspathElements>
+            <additionalClasspathElement>${project.build.testSourceDirectory}/test/resources</additionalClasspathElement>
+          </additionalClasspathElements>
         </configuration>
       </plugin>
       <plugin>
@@ -236,5 +235,26 @@
     <rpm.java.dir>/usr/share/java</rpm.java.dir>
     <rpm.javadoc.dir>/usr/share/javadoc</rpm.javadoc.dir>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <fork>false</fork>
   </properties>
+  <profiles>
+    <profile>
+      <id>Java 8</id>
+      <activation>
+        <jdk>1.8</jdk>
+      </activation>
+      <properties>
+        <fork>true</fork>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>com.sun</groupId>
+          <artifactId>tools</artifactId>
+          <version>1.5.0</version>
+          <scope>system</scope>
+          <systemPath>${java.home}/../lib/tools.jar</systemPath>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>

src/nu/validator/htmlparser/io/HtmlInputStreamReader.java

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2007 Henri Sivonen
- * Copyright (c) 2013 Mozilla Foundation
+ * Copyright (c) 2013-2020 Mozilla Foundation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a 
  * copy of this software and associated documentation files (the "Software"), 
@@ -61,7 +61,7 @@
 public final class HtmlInputStreamReader extends Reader implements
         ByteReadable, Locator, Locator2 {
 
-    private static final int SNIFFING_LIMIT = 1024;
+    private int sniffingLimit = 1024;
 
     private final InputStream inputStream;
 
@@ -87,11 +87,9 @@ public final class HtmlInputStreamReader extends Reader implements
 
     private boolean charsetBoundaryPassed = false;
 
-    private final byte[] byteArray = new byte[4096]; // Length must be >=
+    private byte[] byteArray = new byte[4096]; // Length must be >= sniffingLimit
 
-    // SNIFFING_LIMIT
-
-    private final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray);
+    private ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray);
 
     private boolean needToNotifyTokenizer = false;
 
@@ -112,18 +110,27 @@ public final class HtmlInputStreamReader extends Reader implements
     /**
      * @param inputStream
      * @param errorHandler
-     * @param locator
+     * @param tokenizer
+     * @param driver
+     * @param heuristics
+     * @param sniffingLimit
      * @throws IOException
      * @throws SAXException
      */
     public HtmlInputStreamReader(InputStream inputStream,
             ErrorHandler errorHandler, Tokenizer tokenizer, Driver driver,
-            Heuristics heuristics) throws SAXException, IOException {
+            Heuristics heuristics, int sniffingLimit)
+            throws SAXException, IOException {
         this.inputStream = inputStream;
         this.errorHandler = errorHandler;
         this.tokenizer = tokenizer;
         this.driver = driver;
         this.sniffing = true;
+        if (sniffingLimit != -1) {
+            this.sniffingLimit = sniffingLimit;
+            this.byteArray = new byte[sniffingLimit];
+            this.byteBuffer = ByteBuffer.wrap(byteArray);
+        }
         Encoding encoding = (new BomSniffer(this)).sniff();
         if (encoding == null) {
             position = 0;
@@ -178,6 +185,12 @@ public HtmlInputStreamReader(InputStream inputStream,
         initDecoder();
     }
 
+    public HtmlInputStreamReader(InputStream inputStream,
+            ErrorHandler errorHandler, Tokenizer tokenizer, Driver driver,
+            Heuristics heuristics) throws SAXException, IOException {
+        this(inputStream, errorHandler, tokenizer, driver, heuristics, -1);
+    }
+
     /**
      * 
      */
@@ -237,7 +250,7 @@ public HtmlInputStreamReader(InputStream inputStream,
                 if (charsetBoundaryPassed) {
                     readLen = byteArray.length - oldLimit;
                 } else {
-                    readLen = SNIFFING_LIMIT - oldLimit;
+                    readLen = sniffingLimit - oldLimit;
                 }
                 int num = inputStream.read(byteArray, oldLimit, readLen);
                 if (num == -1) {
@@ -261,7 +274,7 @@ public HtmlInputStreamReader(InputStream inputStream,
                 } else if (cr == CoderResult.UNDERFLOW) {
                     int remaining = byteBuffer.remaining();
                     if (!charsetBoundaryPassed) {
-                        if (bytesRead + remaining >= SNIFFING_LIMIT) {
+                        if (bytesRead + remaining >= sniffingLimit) {
                             needToNotifyTokenizer = true;
                             charsetBoundaryPassed = true;
                         }
@@ -389,12 +402,12 @@ public int readByte() throws IOException {
             throw new IllegalStateException(
                     "readByte() called when not in the sniffing state.");
         }
-        if (position == SNIFFING_LIMIT) {
+        if (position == sniffingLimit) {
             return -1;
         } else if (position < limit) {
             return byteArray[position++] & 0xFF;
         } else {
-            int num = inputStream.read(byteArray, limit, SNIFFING_LIMIT - limit);
+            int num = inputStream.read(byteArray, limit, sniffingLimit - limit);
             if (num == -1) {
                 return -1;
             } else {

test-src/nu/validator/htmlparser/test/EncodingTester.java

@@ -36,6 +36,10 @@
 
 public class EncodingTester {
 
+    static int exitStatus = 0;
+
+    protected static int SNIFFING_LIMIT = 16384;
+
     private final InputStream aggregateStream;
 
     private final StringBuilder builder = new StringBuilder();
@@ -47,7 +51,7 @@ public EncodingTester(InputStream aggregateStream) {
         this.aggregateStream = aggregateStream;
     }
 
-    private void runTests() throws IOException, SAXException {
+    void runTests() throws IOException, SAXException {
         while (runTest()) {
             // spin
         }
@@ -59,10 +63,11 @@ private boolean runTest() throws IOException, SAXException {
         }
         UntilHashInputStream stream = new UntilHashInputStream(aggregateStream);
         HtmlInputStreamReader reader = new HtmlInputStreamReader(stream, null,
-                null, null, Heuristics.NONE);
+                null, null, Heuristics.NONE, SNIFFING_LIMIT);
         Charset charset = reader.getCharset();
         stream.close();
         if (skipLabel()) {
+            exitStatus = 1;
             System.err.println("Premature end of test data.");
             return false;
         }
@@ -73,6 +78,7 @@ private boolean runTest() throws IOException, SAXException {
                 case '\n':
                     break loop;
                 case -1:
+                    exitStatus = 1;
                     System.err.println("Premature end of test data.");
                     return false;
                 default:
@@ -82,9 +88,9 @@ private boolean runTest() throws IOException, SAXException {
         String sniffed = charset.name();
         String expected = Encoding.forName(builder.toString()).newDecoder().charset().name();
         if (expected.equalsIgnoreCase(sniffed)) {
-            System.err.println("Success.");
             // System.err.println(stream);
         } else {
+            exitStatus = 1;
             System.err.println("Failure. Expected: " + expected + " got "
                     + sniffed + ".");
             System.err.println(stream);
@@ -118,6 +124,7 @@ public static void main(String[] args) throws IOException, SAXException {
                     args[i]));
             tester.runTests();
         }
+        System.exit(exitStatus);
     }
 
 }