Minor: Adjusts code fragments to not rely on platform defaults

rzo1 · rzo1 · commit 71375ccc6076 · 2022-01-24T13:45:46.000+01:00
diff --git a/crawler4j-commons/src/main/java/edu/uci/ics/crawler4j/util/Util.java b/crawler4j-commons/src/main/java/edu/uci/ics/crawler4j/util/Util.java
@@ -19,6 +19,8 @@
  */
 package edu.uci.ics.crawler4j.util;
 
+import java.util.Locale;
+
 /**
  * @author Yasser Ganjisaffar
  */
@@ -69,20 +71,20 @@ public static long byteArray2Long(byte[] b) {
     }
 
     public static boolean hasBinaryContent(String contentType) {
-        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
+        String typeStr = (contentType != null) ? contentType.toLowerCase(Locale.ROOT) : "";
 
         return typeStr.contains("image") || typeStr.contains("audio") ||
                typeStr.contains("video") || typeStr.contains("application");
     }
 
     public static boolean hasPlainTextContent(String contentType) {
-        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
+        String typeStr = (contentType != null) ? contentType.toLowerCase(Locale.ROOT) : "";
 
         return typeStr.contains("text") && !typeStr.contains("html");
     }
 
     public static boolean hasCssTextContent(String contentType) {
-        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
+        String typeStr = (contentType != null) ? contentType.toLowerCase(Locale.ROOT) : "";
 
         return typeStr.contains("css");
     }
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/crawler/Page.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/crawler/Page.java
@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 
 import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.Header;
@@ -180,7 +181,7 @@ public void load(HttpEntity entity, int maxBytes) throws IOException {
         }
 
         if (charset != null) {
-            contentCharset = charset.displayName();
+            contentCharset = charset.displayName(Locale.ROOT);
         }
 
         contentData = toByteArray(entity, maxBytes);
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/AllTagMapper.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/AllTagMapper.java
@@ -21,6 +21,8 @@
 
 import org.apache.tika.parser.html.HtmlMapper;
 
+import java.util.Locale;
+
 /**
  * Maps all HTML tags (not ignore some of this)
  *
@@ -30,7 +32,7 @@ public class AllTagMapper implements HtmlMapper {
 
     @Override
     public String mapSafeElement(String name) {
-        return name.toLowerCase();
+        return name.toLowerCase(Locale.ROOT);
     }
 
     @Override
@@ -40,6 +42,6 @@ public boolean isDiscardElement(String name) {
 
     @Override
     public String mapSafeAttribute(String elementName, String attributeName) {
-        return attributeName.toLowerCase();
+        return attributeName.toLowerCase(Locale.ROOT);
     }
 }
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/BinaryParseData.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/BinaryParseData.java
@@ -25,6 +25,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -102,7 +103,7 @@ private static TransformerHandler getTransformerHandler(OutputStream out, String
             transformer.setOutputProperty(OutputKeys.ENCODING, encoding);
         }
 
-        transformerHandler.setResult(new StreamResult(new PrintStream(out)));
+        transformerHandler.setResult(new StreamResult(new PrintStream(out, false, StandardCharsets.UTF_8)));
         return transformerHandler;
     }
 
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/HtmlContentHandler.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/HtmlContentHandler.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import org.xml.sax.Attributes;
@@ -52,7 +53,7 @@ private static class HtmlFactory {
         static {
             name2Element = new HashMap<>();
             for (Element element : Element.values()) {
-                name2Element.put(element.toString().toLowerCase(), element);
+                name2Element.put(element.toString().toLowerCase(Locale.ROOT), element);
             }
         }
 
@@ -118,12 +119,12 @@ public void startElement(String uri, String localName, String qName, Attributes
 
             String content = attributes.getValue("content");
             if ((equiv != null) && (content != null)) {
-                equiv = equiv.toLowerCase();
+                equiv = equiv.toLowerCase(Locale.ROOT);
                 metaTags.put(equiv, content);
 
                 // http-equiv="refresh" content="0;URL=http://foo.bar/..."
                 if ("refresh".equals(equiv) && (metaRefresh == null)) {
-                    int pos = content.toLowerCase().indexOf("url=");
+                    int pos = content.toLowerCase(Locale.ROOT).indexOf("url=");
                     if (pos != -1) {
                         metaRefresh = content.substring(pos + 4);
                         addToOutgoingUrls(metaRefresh, localName);
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/Parser.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/Parser.java
@@ -32,6 +32,7 @@
 import edu.uci.ics.crawler4j.util.Util;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 /**
  * @author Yasser Ganjisaffar
@@ -93,7 +94,7 @@ public void parse(Page page, String contextURL) throws NotAllowedContentExceptio
             try {
                 CssParseData parseData = new CssParseData(factory, normalizer);
                 if (page.getContentCharset() == null) {
-                    parseData.setTextContent(new String(page.getContentData()));
+                    parseData.setTextContent(new String(page.getContentData(), StandardCharsets.UTF_8));
                 } else {
                     parseData.setTextContent(
                             new String(page.getContentData(), page.getContentCharset()));
@@ -108,7 +109,7 @@ public void parse(Page page, String contextURL) throws NotAllowedContentExceptio
             try {
                 TextParseData parseData = new TextParseData();
                 if (page.getContentCharset() == null) {
-                    parseData.setTextContent(new String(page.getContentData()));
+                    parseData.setTextContent(new String(page.getContentData(), StandardCharsets.UTF_8));
                 } else {
                     parseData.setTextContent(
                             new String(page.getContentData(), page.getContentCharset()));
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java
@@ -22,7 +22,9 @@
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Set;
 
 import crawlercommons.filters.basic.BasicURLNormalizer;
@@ -93,7 +95,7 @@ public HtmlParseData parse(Page page, String contextURL) throws ParseException {
             parsedData.setOutgoingUrls(outgoingUrls);
 
             if (page.getContentCharset() == null) {
-                parsedData.setHtml(new String(page.getContentData()));
+                parsedData.setHtml(new String(page.getContentData(), StandardCharsets.UTF_8));
             } else {
                 parsedData.setHtml(new String(page.getContentData(), page.getContentCharset()));
             }
@@ -123,7 +125,7 @@ private Set<WebURL> getOutgoingUrls(String contextURL, HtmlContentHandler conten
                 continue;
             }
 
-            String hrefLoweredCase = href.trim().toLowerCase();
+            String hrefLoweredCase = href.trim().toLowerCase(Locale.ROOT);
             if (!hrefLoweredCase.contains("javascript:") &&
                     !hrefLoweredCase.contains("mailto:") && !hrefLoweredCase.contains("@")) {
                 String url = normalizer.filter(UrlResolver.resolveUrl((contextURL == null) ? "" : contextURL, href));
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtServer.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/robotstxt/RobotstxtServer.java
@@ -44,6 +44,7 @@
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 public class RobotstxtServer {
@@ -70,7 +71,7 @@ public RobotstxtServer(RobotstxtConfig config, PageFetcher pageFetcher, WebURLFa
     }
 
     private static String getHost(URL url) {
-        return url.getHost().toLowerCase();
+        return url.getHost().toLowerCase(Locale.ROOT);
     }
 
     /**
diff --git a/crawler4j-core/src/test/java/edu/uci/ics/crawler4j/tests/fetcher/PageFetcherHtmlOnly.java b/crawler4j-core/src/test/java/edu/uci/ics/crawler4j/tests/fetcher/PageFetcherHtmlOnly.java
@@ -25,6 +25,7 @@
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Date;
+import java.util.Locale;
 
 import crawlercommons.filters.basic.BasicURLNormalizer;
 import edu.uci.ics.crawler4j.url.WebURL;
@@ -70,7 +71,7 @@ public PageFetchResult fetchPage(WebURL webUrl)
 
             String contentType = response.containsHeader("Content-Type") ?
                     response.getFirstHeader("Content-Type").getValue() : null;
-            String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
+            String typeStr = (contentType != null) ? contentType.toLowerCase(Locale.ROOT) : "";
 
             if (typeStr.equals("") || (typeStr.contains("text") && typeStr.contains("html"))) {
                 return super.fetchPage(webUrl);
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/basic/BasicCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/basic/BasicCrawler.java
@@ -19,6 +19,7 @@
  */
 package edu.uci.ics.crawler4j.examples.basic;
 
+import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
@@ -52,7 +53,7 @@ public BasicCrawler(AtomicInteger numSeenImages) {
      */
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         // Ignore the url if it has an extension that matches our defined set of image extensions.
         if (IMAGE_EXTENSIONS.matcher(href).matches()) {
             numSeenImages.incrementAndGet();
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/hsqldb/basic/BasicWikipediaCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/hsqldb/basic/BasicWikipediaCrawler.java
@@ -25,6 +25,7 @@
 import edu.uci.ics.crawler4j.url.WebURL;
 import org.apache.hc.core5.http.Header;
 
+import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
@@ -52,7 +53,7 @@ public BasicWikipediaCrawler(AtomicInteger numSeenImages) {
      */
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         // Ignore the url if it has an extension that matches our defined set of image extensions.
         if (IMAGE_EXTENSIONS.matcher(href).matches()) {
             numSeenImages.incrementAndGet();
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/imagecrawler/ImageCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/imagecrawler/ImageCrawler.java
@@ -26,6 +26,7 @@
 import java.nio.file.StandardOpenOption;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.UUID;
 import java.util.regex.Pattern;
 
@@ -58,7 +59,7 @@ public ImageCrawler(File storageFolder, List<String> crawlDomains) {
 
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         if (filters.matcher(href).matches()) {
             return false;
         }
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/localdata/LocalDataCollectorCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/localdata/LocalDataCollectorCrawler.java
@@ -19,7 +19,8 @@
  */
 package edu.uci.ics.crawler4j.examples.localdata;
 
-import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -46,7 +47,7 @@ public LocalDataCollectorCrawler() {
 
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         return !FILTERS.matcher(href).matches() && href.startsWith("https://www.ics.uci.edu/");
     }
 
@@ -59,11 +60,7 @@ public void visit(Page page) {
             HtmlParseData parseData = (HtmlParseData) page.getParseData();
             Set<WebURL> links = parseData.getOutgoingUrls();
             myCrawlStat.incTotalLinks(links.size());
-            try {
-                myCrawlStat.incTotalTextSize(parseData.getText().getBytes("UTF-8").length);
-            } catch (UnsupportedEncodingException ignored) {
-                // Do nothing
-            }
+            myCrawlStat.incTotalTextSize(parseData.getText().getBytes(StandardCharsets.UTF_8).length);
         }
         // We dump this crawler statistics after processing every 50 pages
         if ((myCrawlStat.getTotalProcessedPages() % 50) == 0) {
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/multiple/BasicCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/multiple/BasicCrawler.java
@@ -21,6 +21,7 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -47,7 +48,7 @@ public BasicCrawler(List<String> myCrawlDomains) {
 
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         if (FILTERS.matcher(href).matches()) {
             return false;
         }
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/shutdown/BasicCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/shutdown/BasicCrawler.java
@@ -19,6 +19,7 @@
  */
 package edu.uci.ics.crawler4j.examples.shutdown;
 
+import java.util.Locale;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -45,7 +46,7 @@ public class BasicCrawler extends WebCrawler {
 
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         return !FILTERS.matcher(href).matches() && href.startsWith(DOMAIN);
     }
 
diff --git a/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/statushandler/StatusHandlerCrawler.java b/crawler4j-examples/crawler4j-examples-base/src/test/java/edu/uci/ics/crawler4j/examples/statushandler/StatusHandlerCrawler.java
@@ -19,6 +19,7 @@
  */
 package edu.uci.ics.crawler4j.examples.statushandler;
 
+import java.util.Locale;
 import java.util.regex.Pattern;
 
 import org.apache.hc.core5.http.HttpStatus;
@@ -46,7 +47,7 @@ public class StatusHandlerCrawler extends WebCrawler {
      */
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         return !FILTERS.matcher(href).matches() && href.startsWith("https://www.ics.uci.edu/");
     }
 
diff --git a/crawler4j-examples/crawler4j-examples-postgres/src/main/java/edu/uci/ics/crawler4j/examples/crawler/PostgresWebCrawler.java b/crawler4j-examples/crawler4j-examples-postgres/src/main/java/edu/uci/ics/crawler4j/examples/crawler/PostgresWebCrawler.java
@@ -19,6 +19,7 @@
  */
 package edu.uci.ics.crawler4j.examples.crawler;
 
+import java.util.Locale;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -53,7 +54,7 @@ public PostgresWebCrawler(PostgresDBService postgresDBService) {
 
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
-        String href = url.getURL().toLowerCase();
+        String href = url.getURL().toLowerCase(Locale.ROOT);
         return !FILE_ENDING_EXCLUSION_PATTERN.matcher(href).matches();
     }
 
diff --git a/crawler4j-frontier/crawler4j-frontier-sleepycat/src/main/java/edu/uci/ics/crawler4j/frontier/Counters.java b/crawler4j-frontier/crawler4j-frontier-sleepycat/src/main/java/edu/uci/ics/crawler4j/frontier/Counters.java
@@ -19,6 +19,7 @@
  */
 package edu.uci.ics.crawler4j.frontier;
 
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -83,7 +84,7 @@ public Counters(Environment env, CrawlConfig config) {
 
             while (result == OperationStatus.SUCCESS) {
                 if (value.getData().length > 0) {
-                    String name = new String(key.getData());
+                    String name = new String(key.getData(), StandardCharsets.UTF_8);
                     long counterValue = Util.byteArray2Long(value.getData());
                     counterValues.put(name, counterValue);
                 }
@@ -110,7 +111,7 @@ public void setValue(String name, long value) {
                 counterValues.put(name, value);
                 if (statisticsDB != null) {
                     Transaction txn = env.beginTransaction(null, null);
-                    statisticsDB.put(txn, new DatabaseEntry(name.getBytes()),
+                    statisticsDB.put(txn, new DatabaseEntry(name.getBytes(StandardCharsets.UTF_8)),
                                      new DatabaseEntry(Util.long2ByteArray(value)));
                     txn.commit();
                 }
diff --git a/crawler4j-frontier/crawler4j-frontier-sleepycat/src/main/java/edu/uci/ics/crawler4j/frontier/SleepycatDocIDServer.java b/crawler4j-frontier/crawler4j-frontier-sleepycat/src/main/java/edu/uci/ics/crawler4j/frontier/SleepycatDocIDServer.java
diff --git a/pom.xml b/pom.xml

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@`
`23`	`23`	`import java.io.InputStream;`
`24`	`24`	`import java.nio.charset.Charset;`
`25`	`25`	`import java.nio.charset.StandardCharsets;`
	`26`	`+import java.util.Locale;`
`26`	`27`
`27`	`28`	`import org.apache.hc.core5.http.ContentType;`
`28`	`29`	`import org.apache.hc.core5.http.Header;`
`@@ -180,7 +181,7 @@ public void load(HttpEntity entity, int maxBytes) throws IOException {`
`180`	`181`	`}`
`181`	`182`
`182`	`183`	`if (charset != null) {`
`183`		`- contentCharset = charset.displayName();`
	`184`	`+ contentCharset = charset.displayName(Locale.ROOT);`
`184`	`185`	`}`
`185`	`186`
`186`	`187`	`contentData = toByteArray(entity, maxBytes);`
Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,8 @@`
`21`	`21`
`22`	`22`	`import org.apache.tika.parser.html.HtmlMapper;`
`23`	`23`
	`24`	`+import java.util.Locale;`
	`25`	`+`
`24`	`26`	`/**`
`25`	`27`	`* Maps all HTML tags (not ignore some of this)`
`26`	`28`	`*`
`@@ -30,7 +32,7 @@ public class AllTagMapper implements HtmlMapper {`
`30`	`32`
`31`	`33`	`@Override`
`32`	`34`	`public String mapSafeElement(String name) {`
`33`		`- return name.toLowerCase();`
	`35`	`+ return name.toLowerCase(Locale.ROOT);`
`34`	`36`	`}`
`35`	`37`
`36`	`38`	`@Override`
`@@ -40,6 +42,6 @@ public boolean isDiscardElement(String name) {`
`40`	`42`
`41`	`43`	`@Override`
`42`	`44`	`public String mapSafeAttribute(String elementName, String attributeName) {`
`43`		`- return attributeName.toLowerCase();`
	`45`	`+ return attributeName.toLowerCase(Locale.ROOT);`
`44`	`46`	`}`
`45`	`47`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@`
`25`	`25`	`import java.io.InputStream;`
`26`	`26`	`import java.io.OutputStream;`
`27`	`27`	`import java.io.PrintStream;`
	`28`	`+import java.nio.charset.StandardCharsets;`
`28`	`29`	`import java.util.HashSet;`
`29`	`30`	`import java.util.Set;`
`30`	`31`
`@@ -102,7 +103,7 @@ private static TransformerHandler getTransformerHandler(OutputStream out, String`
`102`	`103`	`transformer.setOutputProperty(OutputKeys.ENCODING, encoding);`
`103`	`104`	`}`
`104`	`105`
`105`		`- transformerHandler.setResult(new StreamResult(new PrintStream(out)));`
	`106`	`+ transformerHandler.setResult(new StreamResult(new PrintStream(out, false, StandardCharsets.UTF_8)));`
`106`	`107`	`return transformerHandler;`
`107`	`108`	`}`
`108`	`109`
Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,7 @@`
`44`	`44`	`import java.net.URL;`
`45`	`45`	`import java.net.UnknownHostException;`
`46`	`46`	`import java.util.HashMap;`
	`47`	`+import java.util.Locale;`
`47`	`48`	`import java.util.Map;`
`48`	`49`
`49`	`50`	`public class RobotstxtServer {`
`@@ -70,7 +71,7 @@ public RobotstxtServer(RobotstxtConfig config, PageFetcher pageFetcher, WebURLFa`
`70`	`71`	`}`
`71`	`72`
`72`	`73`	`private static String getHost(URL url) {`
`73`		`- return url.getHost().toLowerCase();`
	`74`	`+ return url.getHost().toLowerCase(Locale.ROOT);`
`74`	`75`	`}`
`75`	`76`
`76`	`77`	`/**`