TIKA-4016 -- add length in InputStreamDigester (#1303)

apache · Aug 24, 2023 · a2f1a19 · a2f1a19
1 parent 97a1b19
commit a2f1a19
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 8 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,3 +1,7 @@
+Release 2.9.1 - ??
+
+   * The InputStreamDigester now calculates stream length (TIKA-4016).
+
 Release 2.9.0 - 8/23/2023
 
    * With user configuration, the PDFParser can now throw an EncryptedDocumentException

diff --git a/tika-core/src/main/java/org/apache/tika/parser/digest/InputStreamDigester.java b/tika-core/src/main/java/org/apache/tika/parser/digest/InputStreamDigester.java
@@ -33,6 +33,7 @@
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.DigestingParser;
 import org.apache.tika.parser.ParseContext;
+import org.apache.tika.utils.StringUtils;
 
 public class InputStreamDigester implements DigestingParser.Digester {
 
@@ -72,17 +73,25 @@ public InputStreamDigester(int markLimit, String algorithm, String algorithmKeyN
     /**
      * Copied from commons-codec
      */
-    private static MessageDigest updateDigest(MessageDigest digest, InputStream data)
+    private static MessageDigest updateDigest(MessageDigest digest, InputStream data, Metadata metadata)
             throws IOException {
         byte[] buffer = new byte[1024];
-
+        long total = 0;
         for (int read = data.read(buffer, 0, 1024); read > -1; read = data.read(buffer, 0, 1024)) {
             digest.update(buffer, 0, read);
+            total += read;
         }
-
+        setContentLength(total, metadata);
         return digest;
     }
 
+    private static void setContentLength(long length, Metadata metadata) {
+        if (StringUtils.isBlank(metadata.get(Metadata.CONTENT_LENGTH))) {
+            //only add it if it hasn't been populated already
+            metadata.set(Metadata.CONTENT_LENGTH, Long.toString(length));
+        }
+    }
+
     private MessageDigest newMessageDigest() {
         try {
             Provider provider = getProvider();
@@ -128,7 +137,7 @@ public void digest(InputStream is, Metadata metadata, ParseContext parseContext)
             //and its size is greater than its mark limit,
             //just digest the underlying file.
             if (sz > markLimit) {
-                digestFile(tis.getFile(), metadata);
+                digestFile(tis.getFile(), sz, metadata);
                 return;
             }
         }
@@ -148,12 +157,12 @@ public void digest(InputStream is, Metadata metadata, ParseContext parseContext)
         //if the stream wasn't finished -- if the stream was longer than the mark limit --
         //spool to File and digest that.
         if (tis != null) {
-            digestFile(tis.getFile(), metadata);
+            digestFile(tis.getFile(), -1, metadata);
         } else {
             TemporaryResources tmp = new TemporaryResources();
             try {
                 TikaInputStream tmpTikaInputStream = TikaInputStream.get(is, tmp, metadata);
-                digestFile(tmpTikaInputStream.getFile(), metadata);
+                digestFile(tmpTikaInputStream.getFile(), -1, metadata);
             } finally {
                 try {
                     tmp.dispose();
@@ -169,7 +178,14 @@ private String getMetadataKey() {
                 TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER + algorithmKeyName;
     }
 
-    private void digestFile(File f, Metadata m) throws IOException {
+    private void digestFile(File f, long sz, Metadata m) throws IOException {
+        //only add it if it hasn't been populated already
+        if (StringUtils.isBlank(m.get(Metadata.CONTENT_LENGTH))) {
+            if (sz < 0) {
+                sz = f.length();
+            }
+            setContentLength(sz, m);
+        }
         try (InputStream is = new FileInputStream(f)) {
             digestStream(is, m);
         }
@@ -185,7 +201,7 @@ private boolean digestStream(InputStream is, Metadata metadata) throws IOExcepti
         byte[] digestBytes;
         MessageDigest messageDigest = newMessageDigest();
 
-        updateDigest(messageDigest, is);
+        updateDigest(messageDigest, is, metadata);
         digestBytes = messageDigest.digest();
 
         if (is instanceof BoundedInputStream) {

diff --git a/...ka-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java b/...ka-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java
@@ -30,13 +30,15 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.junit.jupiter.api.Test;
 
 import org.apache.tika.TikaTest;
+import org.apache.tika.config.TikaConfig;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
@@ -115,6 +117,21 @@ public void testCommaSeparated() throws Exception {
         assertNull(m.get(P + CommonsDigester.DigestAlgorithm.MD2.toString()));
     }
 
+    @Test
+    public void testLengthsCalculated() throws Exception {
+        //This tests that TIKA-4016 added lengths
+        //before TIKA-4016, lengths were missing from 0, 1 and 11
+        TikaConfig config = null;
+        try (InputStream is = getResourceAsStream("/configs/tika-config-digests.xml")) {
+            config = new TikaConfig(is);
+        }
+        Parser p = new AutoDetectParser(config);
+        List<Metadata> metadataList = getRecursiveMetadata("test_recursive_embedded.docx", p);
+        for (Metadata m : metadataList) {
+            assertNotNull(m.get(Metadata.CONTENT_LENGTH));
+        }
+    }
+
     @Test
     public void testReset() throws Exception {
         String expectedMD5 = "59f626e09a8c16ab6dbc2800c685f772";