Support more archive formats (cpio, ar) (#64)

src/main/java/io/github/zlika/reproducible/ArchiveStripper.java

@@ -0,0 +1,191 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.github.zlika.reproducible;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.ArchiveOutputStream;
+import org.apache.commons.compress.archivers.ArchiveStreamFactory;
+import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.CompressorException;
+
+/**
+ * Strip archives of file dates and users,groups informations that are not reproducible.
+ */
+public class ArchiveStripper implements Stripper
+{
+    private final long timestamp;
+
+    /**
+     * Constructor.
+     * @param reproducibleDateTime the date/time to use in TAR entries.
+     */
+    public ArchiveStripper(LocalDateTime reproducibleDateTime)
+    {
+        this.timestamp = reproducibleDateTime.atZone(ZoneOffset.UTC).toInstant().toEpochMilli();
+    }
+
+    @Override
+    public void strip(File in, File out) throws IOException
+    {
+        Path tmp = Files.createTempDirectory("tmp-" + in.getName());
+
+        try (InputStream is = new BufferedInputStream(new FileInputStream(in));
+             OutputStream os = new BufferedOutputStream(new FileOutputStream(out)))
+        {
+            strip(is, os, tmp);
+        }
+        catch (ArchiveException | CompressorException e)
+        {
+            throw new IOException(e);
+        }
+        finally
+        {
+            org.codehaus.plexus.util.FileUtils.deleteDirectory(tmp.toFile());
+        }
+    }
+
+    void strip(InputStream in, OutputStream out, Path tmp)
+            throws IOException, ArchiveException, CompressorException
+    {
+        String format = ArchiveStreamFactory.detect(in);
+        try (ArchiveInputStream ain = ArchiveStreamFactory.DEFAULT.createArchiveInputStream(format, in);
+             ArchiveOutputStream aout = ArchiveStreamFactory.DEFAULT.createArchiveOutputStream(format, out))
+        {
+            if (aout instanceof TarArchiveOutputStream)
+            {
+                ((TarArchiveOutputStream) aout).setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+            }
+            strip(ain, aout, tmp);
+        }
+    }
+
+    void strip(ArchiveInputStream ain, ArchiveOutputStream aout, Path tmp) throws IOException
+    {
+        List<ArchiveEntry> sortedNames = new ArrayList<>();
+        ArchiveEntry entry;
+        while ((entry = ain.getNextEntry()) != null)
+        {
+            sortedNames.add(entry);
+            File copyTo = new File(tmp.toFile(), entry.getName());
+            zipSlipProtection(copyTo, tmp);
+            if (entry.isDirectory())
+            {
+                FileUtils.mkdirs(copyTo);
+            }
+            else
+            {
+                File destParent = copyTo.getParentFile();
+                FileUtils.mkdirs(destParent);
+                Files.copy(ain, copyTo.toPath());
+            }
+        }
+
+        sortedNames.sort(Comparator.comparing(ArchiveEntry::getName));
+
+        for (ArchiveEntry sortedEntry : sortedNames)
+        {
+            File copyFrom = new File(tmp.toFile(), sortedEntry.getName());
+            if (!sortedEntry.isDirectory())
+            {
+                byte[] fileContent = Files.readAllBytes(copyFrom.toPath());
+                if (sortedEntry instanceof TarArchiveEntry)
+                {
+                    TarArchiveEntry tarEntry = (TarArchiveEntry) sortedEntry;
+                    tarEntry.setSize(fileContent.length);
+                }
+                else if (sortedEntry instanceof ArArchiveEntry)
+                {
+                    ArArchiveEntry arEntry = (ArArchiveEntry) sortedEntry;
+                    sortedEntry = new ArArchiveEntry(arEntry.getName(), fileContent.length, arEntry.getUserId(),
+                            arEntry.getGroupId(), arEntry.getMode(), arEntry.getLastModified());
+                }
+                else if (sortedEntry instanceof CpioArchiveEntry)
+                {
+                    CpioArchiveEntry cpioEntry = (CpioArchiveEntry) sortedEntry;
+                    cpioEntry.setSize(fileContent.length);
+                }
+                aout.putArchiveEntry(filterEntry(sortedEntry));
+                aout.write(fileContent);
+                aout.closeArchiveEntry();
+            }
+            else
+            {
+                aout.putArchiveEntry(filterEntry(sortedEntry));
+                aout.closeArchiveEntry();
+            }
+        }
+    }
+
+    private ArchiveEntry filterEntry(ArchiveEntry entry)
+    {
+        if (entry instanceof TarArchiveEntry)
+        {
+            TarArchiveEntry tarEntry = (TarArchiveEntry) entry;
+            tarEntry.setModTime(timestamp);
+            tarEntry.setGroupId(0);
+            tarEntry.setUserId(0);
+            tarEntry.setUserName("");
+            tarEntry.setGroupName("");
+        }
+        else if (entry instanceof ArArchiveEntry)
+        {
+            ArArchiveEntry arEntry = (ArArchiveEntry) entry;
+            return new ArArchiveEntry(arEntry.getName(), arEntry.getSize(), 0, 0, arEntry.getMode(), timestamp / 1000);
+        }
+        else if (entry instanceof CpioArchiveEntry)
+        {
+            CpioArchiveEntry cpioEntry = (CpioArchiveEntry) entry;
+            cpioEntry.setTime(timestamp / 1000);
+            cpioEntry.setUID(0);
+            cpioEntry.setGID(0);
+        }
+        return entry;
+    }
+
+    /**
+     * Protection against 'Zip Slip' vulnerability.
+     * This method checks that the file path is located inside the expected folder.
+     * @param file the file path to check.
+     * @param extractFolder the folder.
+     * @throws IOException if a 'Zip Slip' attack is detected.
+     */
+    private void zipSlipProtection(File file, Path extractFolder) throws IOException
+    {
+        if (!file.toPath().normalize().startsWith(extractFolder))
+        {
+            throw new IOException("Bad zip entry");
+        }
+    }
+}

src/main/java/io/github/zlika/reproducible/CompressedArchiveStripper.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.github.zlika.reproducible;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+
+/**
+ * Stripper implementation for compressed archives.
+ */
+public class CompressedArchiveStripper extends ArchiveStripper
+{
+    /**
+     * Constructor.
+     * @param reproducibleDateTime the date/time to use in the archive entries.
+     */
+    public CompressedArchiveStripper(LocalDateTime reproducibleDateTime)
+    {
+        super(reproducibleDateTime);
+    }
+
+    @Override
+    void strip(InputStream in, OutputStream out, Path tmp)
+            throws IOException, ArchiveException, CompressorException
+    {
+        String format = CompressorStreamFactory.detect(in);
+
+        CompressorStreamFactory compressorFactory = CompressorStreamFactory.getSingleton();
+        try (InputStream cis = new BufferedInputStream(compressorFactory.createCompressorInputStream(format, in));
+             OutputStream cout = new BufferedOutputStream(compressorFactory.createCompressorOutputStream(format, out)))
+        {
+            super.strip(cis, cout, tmp);
+        }
+    }
+}

src/main/java/io/github/zlika/reproducible/SmartTarStripper.java → src/main/java/io/github/zlika/reproducible/SmartArchiveStripper.java

@@ -18,19 +18,19 @@
 import java.time.LocalDateTime;
 
 /**
- * Process tar formats: tar, tar.gz, tar.bz2 using the default configuriaton
- * and the right tar stripper implementation.
+ * Process archive formats: tar, tar.gz, tar.bz2, ar, cpio using the default configuration
+ * and the right stripper implementation.
  * @author Umberto Nicoletti ([email protected])
  */
-final class SmartTarStripper implements Stripper
+final class SmartArchiveStripper implements Stripper
 {
     private final LocalDateTime reproducibleDateTime;
 
     /**
      * Constructor.
-     * @param reproducibleDateTime the date/time to use in TAR entries.
+     * @param reproducibleDateTime the date/time to use in the archive entries.
      */
-    public SmartTarStripper(LocalDateTime reproducibleDateTime)
+    public SmartArchiveStripper(LocalDateTime reproducibleDateTime)
     {
         this.reproducibleDateTime = reproducibleDateTime;
     }
@@ -50,19 +50,13 @@ public void strip(final File file, final File stripped) throws IOException
     private Stripper findImplementation(File file)
     {
         final String name = file.getName();
-        final Stripper impl;
-        if (name.endsWith(".tar.gz"))
+        if (name.endsWith(".tar.gz") || name.endsWith(".tar.bz2"))
         {
-            impl = new TarGzStripper(reproducibleDateTime);
-        }
-        else if (name.endsWith(".tar.bz2"))
-        {
-            impl = new TarBzStripper(reproducibleDateTime);
+            return new CompressedArchiveStripper(reproducibleDateTime);
         }
         else
         {
-            impl = new TarStripper(reproducibleDateTime);
+            return new ArchiveStripper(reproducibleDateTime);
         }
-        return impl;
     }
 }

src/main/java/io/github/zlika/reproducible/StripJarMojo.java

@@ -30,16 +30,15 @@
 import org.apache.maven.plugins.annotations.Parameter;
 
 /**
- * Fixes the produced artifacts (ZIP/JAR/WAR/EAR) to make the build reproducible.
+ * Fixes the produced artifacts to make the build reproducible.
  */
 @Mojo(name = "strip-jar", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST,
         requiresProject = false, threadSafe = true)
 public final class StripJarMojo extends AbstractMojo
 {
     private static final List<String> ZIP_EXT = Arrays.asList("zip", "jar", "war", "ear", "hpi", "adapter");
-    private static final List<String>  TAR_GZ_EXT = Collections.singletonList("tar.gz");
-    private static final List<String>  TAR_BZ_EXT = Collections.singletonList("tar.bz2");
-    private static final List<String>  TAR_EXT = Collections.singletonList("tar");
+    private static final List<String> ARCHIVE_EXT =
+            Arrays.asList(".tar", ".tar.gz", ".tar.bz2", ".tgz", ".cpio", ".rpm", ".ar", ".deb");
     private static final byte[] ZIP_FILE_HEADER = new byte[] { 0x50, 0x4B, 0x03, 0x04 };
     private static final byte[] SPRING_BOOT_EXEC_HEADER = new byte[] { 0x23, 0x21, 0x2F, 0x62, 0x69, 0x6E };
 
@@ -181,16 +180,8 @@ public void execute() throws MojoExecutionException
                                 new DefaultZipStripper(zipStripper, this.manifestAttributes)))
             );
             this.process(
-                this.findTarFiles(this.outputDirectory),
-                new OverwriteStripper(this.overwrite, new SmartTarStripper(reproducibleDateTime))
-            );
-            this.process(
-                this.findTarBzFiles(this.outputDirectory),
-                new OverwriteStripper(this.overwrite, new SmartTarStripper(reproducibleDateTime))
-            );
-            this.process(
-                this.findTarGzFiles(this.outputDirectory),
-                new OverwriteStripper(this.overwrite, new SmartTarStripper(reproducibleDateTime))
+                this.findArchiveFiles(this.outputDirectory),
+                        new OverwriteStripper(this.overwrite, new SmartArchiveStripper(reproducibleDateTime))
             );
         }
     }
@@ -267,28 +258,12 @@ private byte[] getFileHeader(final File file, final int length)
         return header;
     }
 
-    private File[] findTarBzFiles(final File folder)
-    {
-        final PatternFileNameFilter filter =
-                PatternFileNameFilter.of(this.getLog(), this.includes, this.excludes, TAR_BZ_EXT);
-        final File[] tbzFiles = folder.listFiles(filter);
-        return tbzFiles != null ? tbzFiles : new File[0];
-    }
-
-    private File[] findTarGzFiles(final File folder)
-    {
-        final PatternFileNameFilter filter =
-                PatternFileNameFilter.of(this.getLog(), this.includes, this.excludes, TAR_GZ_EXT);
-        final File[] tgzFiles = folder.listFiles(filter);
-        return tgzFiles != null ? tgzFiles : new File[0];
-    }
-
-    private File[] findTarFiles(final File folder)
+    private File[] findArchiveFiles(final File folder)
     {
         final PatternFileNameFilter filter =
-                PatternFileNameFilter.of(this.getLog(), this.includes, this.excludes, TAR_EXT);
-        final File[] tarFiles = folder.listFiles(filter);
-        return tarFiles != null ? tarFiles : new File[0];
+                PatternFileNameFilter.of(this.getLog(), this.includes, this.excludes, ARCHIVE_EXT);
+        final File[] archiveFiles = folder.listFiles(filter);
+        return archiveFiles != null ? archiveFiles : new File[0];
     }
 
     private File createStrippedFilename(final File originalFile)