Skip to content

Commit 30a6a70

Browse files
Zlikalaeubi
Zlika
authored and
laeubi
committed
Write Properties files in a reproducible way
1 parent e052363 commit 30a6a70

File tree

13 files changed

+130
-93
lines changed

13 files changed

+130
-93
lines changed

‎p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.codehaus.plexus.component.annotations.Requirement;
4141
import org.codehaus.plexus.logging.Logger;
4242
import org.eclipse.equinox.internal.p2.repository.AuthenticationFailedException;
43+
import org.eclipse.tycho.ReproducibleUtils;
4344

4445
@Component(role = HttpCache.class)
4546
public class SharedHttpCacheStorage implements HttpCache {
@@ -409,12 +410,9 @@ protected void updateHeader(Headers response, int code) throws IOException, File
409410
header.put(key, value.stream().collect(Collectors.joining(",")));
410411
}
411412
}
412-
FileUtils.forceMkdir(file.getParentFile());
413-
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(headerFile))) {
414-
// we store the header here, this might be a 404 response or (permanent)
415-
// redirect we probably need to work with later on
416-
header.store(out, null);
417-
}
413+
// we store the header here, this might be a 404 response or (permanent)
414+
// redirect we probably need to work with later on
415+
ReproducibleUtils.storeProperties(header, headerFile.toPath());
418416
}
419417

420418
private synchronized Date pareHttpDate(String input) {

‎sisu-osgi/sisu-equinox-launching/src/main/java/org/eclipse/sisu/equinox/launching/internal/DefaultEquinoxInstallationFactory.java

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,9 @@
1212
*******************************************************************************/
1313
package org.eclipse.sisu.equinox.launching.internal;
1414

15-
import java.io.BufferedOutputStream;
1615
import java.io.File;
1716
import java.io.FileInputStream;
18-
import java.io.FileOutputStream;
1917
import java.io.IOException;
20-
import java.io.OutputStream;
2118
import java.util.ArrayList;
2219
import java.util.Collection;
2320
import java.util.HashMap;
@@ -45,6 +42,7 @@
4542
import org.eclipse.sisu.equinox.launching.EquinoxInstallation;
4643
import org.eclipse.sisu.equinox.launching.EquinoxInstallationDescription;
4744
import org.eclipse.sisu.equinox.launching.EquinoxInstallationFactory;
45+
import org.eclipse.tycho.ReproducibleUtils;
4846
import org.eclipse.tycho.TychoConstants;
4947
import org.osgi.framework.Constants;
5048

@@ -148,12 +146,8 @@ public EquinoxInstallation createInstallation(EquinoxInstallationDescription des
148146
}
149147

150148
File configIni = new File(location, TychoConstants.CONFIG_INI_PATH);
149+
ReproducibleUtils.storeProperties(p, configIni.toPath());
151150
File configurationLocation = configIni.getParentFile();
152-
configurationLocation.mkdirs();
153-
try (OutputStream fos = new BufferedOutputStream(new FileOutputStream(configIni))) {
154-
p.store(fos, null);
155-
}
156-
157151
return new DefaultEquinoxInstallation(description, location, configurationLocation);
158152
} catch (IOException e) {
159153
throw new RuntimeException("Exception creating test eclipse runtime", e);
@@ -210,9 +204,7 @@ private String createDevProperties(File location, Map<String, String> devEntries
210204
File file = new File(location, "dev.properties");
211205
Properties properties = new Properties();
212206
properties.putAll(devEntries);
213-
try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
214-
properties.store(os, null);
215-
}
207+
ReproducibleUtils.storeProperties(properties, file.toPath());
216208
return file.toURI().toURL().toExternalForm();
217209
}
218210

‎tycho-api/src/main/java/org/eclipse/tycho/ReproducibleUtils.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*******************************************************************************
2+
* This program and the accompanying materials
3+
* are made available under the terms of the Eclipse Public License 2.0
4+
* which accompanies this distribution, and is available at
5+
* https://www.eclipse.org/legal/epl-2.0/
6+
*
7+
* SPDX-License-Identifier: EPL-2.0
8+
*******************************************************************************/
9+
package org.eclipse.tycho;
10+
11+
import java.io.BufferedOutputStream;
12+
import java.io.ByteArrayOutputStream;
13+
import java.io.IOException;
14+
import java.io.OutputStream;
15+
import java.nio.charset.StandardCharsets;
16+
import java.nio.file.Files;
17+
import java.nio.file.Path;
18+
import java.util.Properties;
19+
import java.util.stream.Collectors;
20+
21+
/**
22+
* Utility methods for reproducible builds.
23+
*/
24+
public class ReproducibleUtils {
25+
private ReproducibleUtils() {
26+
}
27+
28+
/**
29+
* Writes the property list to the output stream in a reproducible way. The java.util.Properties
30+
* class writes the lines in a non-reproducible order, adds a non-reproducible timestamp and
31+
* uses platform-dependent new line characters.
32+
*
33+
* @param properties
34+
* the properties object to write to the file.
35+
* @param file
36+
* the file to write to. All the missing parent directories are also created.
37+
* @throws IOException
38+
* if writing the property list to the specified output stream throws an
39+
* IOException.
40+
*/
41+
public static void storeProperties(Properties properties, Path file) throws IOException {
42+
final Path folder = file.getParent();
43+
if (folder != null) {
44+
Files.createDirectories(folder);
45+
}
46+
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
47+
properties.store(baos, null);
48+
final String content = baos.toString(StandardCharsets.ISO_8859_1).lines().filter(line -> !line.startsWith("#"))
49+
.sorted().collect(Collectors.joining("\n", "", "\n"));
50+
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(file))) {
51+
os.write(content.getBytes(StandardCharsets.ISO_8859_1));
52+
}
53+
}
54+
}

‎tycho-core/src/main/java/org/eclipse/tycho/p2/repository/module/ModuleArtifactMap.java

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,9 @@
1414

1515
import static org.eclipse.tycho.p2.repository.BundleConstants.BUNDLE_ID;
1616

17-
import java.io.BufferedOutputStream;
1817
import java.io.File;
1918
import java.io.FileInputStream;
20-
import java.io.FileOutputStream;
2119
import java.io.IOException;
22-
import java.io.OutputStream;
2320
import java.util.HashMap;
2421
import java.util.LinkedHashMap;
2522
import java.util.Map;
@@ -29,6 +26,7 @@
2926
import org.eclipse.core.runtime.IStatus;
3027
import org.eclipse.core.runtime.Status;
3128
import org.eclipse.equinox.p2.core.ProvisionException;
29+
import org.eclipse.tycho.ReproducibleUtils;
3230
import org.eclipse.tycho.TychoConstants;
3331
import org.eclipse.tycho.p2.repository.MavenRepositoryCoordinates;
3432
import org.eclipse.tycho.p2.repository.RepositoryReader;
@@ -151,7 +149,7 @@ private void store() throws ProvisionException {
151149
}
152150

153151
try {
154-
writeProperties(outputProperties, mapFile);
152+
ReproducibleUtils.storeProperties(outputProperties, mapFile.toPath());
155153
} catch (IOException e) {
156154
String message = "I/O error while writing repository to " + mapFile;
157155
int code = ProvisionException.REPOSITORY_FAILED_WRITE;
@@ -160,10 +158,4 @@ private void store() throws ProvisionException {
160158
}
161159

162160
}
163-
164-
private static void writeProperties(Properties properties, File outputFile) throws IOException {
165-
try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
166-
properties.store(outputStream, null);
167-
}
168-
}
169161
}

‎tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2GeneratorImpl.java

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,9 @@
1313
*******************************************************************************/
1414
package org.eclipse.tycho.p2resolver;
1515

16-
import java.io.BufferedOutputStream;
1716
import java.io.File;
1817
import java.io.FileInputStream;
19-
import java.io.FileOutputStream;
2018
import java.io.IOException;
21-
import java.io.OutputStream;
2219
import java.util.ArrayList;
2320
import java.util.Arrays;
2421
import java.util.HashMap;
@@ -69,6 +66,7 @@
6966
import org.eclipse.tycho.OptionalResolutionAction;
7067
import org.eclipse.tycho.PackagingType;
7168
import org.eclipse.tycho.ReactorProject;
69+
import org.eclipse.tycho.ReproducibleUtils;
7270
import org.eclipse.tycho.TargetEnvironment;
7371
import org.eclipse.tycho.TychoConstants;
7472
import org.eclipse.tycho.core.osgitools.BundleReader;
@@ -483,13 +481,7 @@ static void writeArtifactLocations(File outputFile, Map<String, File> artifactLo
483481
}
484482
}
485483

486-
writeProperties(outputProperties, outputFile);
487-
}
488-
489-
private static void writeProperties(Properties properties, File outputFile) throws IOException {
490-
try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
491-
properties.store(outputStream, null);
492-
}
484+
ReproducibleUtils.storeProperties(outputProperties, outputFile.toPath());
493485
}
494486

495487
/**

‎tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import java.io.File;
1818
import java.io.FileOutputStream;
1919
import java.io.IOException;
20-
import java.io.OutputStream;
2120
import java.net.HttpURLConnection;
2221
import java.net.URI;
2322
import java.net.URLConnection;
@@ -72,6 +71,7 @@
7271
import org.eclipse.tycho.ArtifactType;
7372
import org.eclipse.tycho.BuildDirectory;
7473
import org.eclipse.tycho.DependencySeed;
74+
import org.eclipse.tycho.ReproducibleUtils;
7575
import org.eclipse.tycho.core.shared.StatusTool;
7676
import org.eclipse.tycho.p2.repository.GAV;
7777
import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -523,9 +523,8 @@ private void writeP2Index(File repositoryDestination) throws FacadeException {
523523
properties.setProperty("version", "1");
524524
properties.setProperty("artifact.repository.factory.order", "artifacts.xml,!");
525525
properties.setProperty("metadata.repository.factory.order", "content.xml,!");
526-
try (OutputStream stream = new BufferedOutputStream(
527-
new FileOutputStream(new File(repositoryDestination, P2_INDEX_FILE)))) {
528-
properties.store(stream, null);
526+
try {
527+
ReproducibleUtils.storeProperties(properties, new File(repositoryDestination, P2_INDEX_FILE).toPath());
529528
} catch (IOException e) {
530529
throw new FacadeException("writing index file failed", e);
531530
}

‎tycho-its/src/test/java/org/eclipse/tycho/test/reproducible/ReproducibleBuildTest.java

Lines changed: 48 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.eclipse.tycho.test.reproducible;
1010

1111
import java.io.IOException;
12+
import java.nio.charset.StandardCharsets;
1213
import java.nio.file.FileSystem;
1314
import java.nio.file.FileSystems;
1415
import java.nio.file.Files;
@@ -25,26 +26,40 @@
2526
import org.junit.Assert;
2627
import org.junit.Test;
2728

29+
/**
30+
* Tests that the build artifacts produced by Tycho are reproducible.
31+
*/
2832
public class ReproducibleBuildTest extends AbstractTychoIntegrationTest {
2933
// The ZipEntry.getLastModifiedTime() method uses the default timezone to
3034
// convert date and time fields to Instant, so we also use the default timezone
3135
// for the expected timestamp here.
3236
private static final String EXPECTED_TIMESTAMP_STRING = "2023-01-01T00:00:00";
3337
private static final Instant EXPECTED_TIMESTAMP_INSTANT = LocalDateTime.parse(EXPECTED_TIMESTAMP_STRING)
3438
.toInstant(OffsetDateTime.now().getOffset());
39+
Verifier verifier;
3540

3641
/**
37-
* Check that the build is reproducible.
42+
* Run the maven integration tests related to reproducible builds.
43+
*
44+
* @throws Exception
3845
*/
3946
@Test
40-
public void test() throws Exception {
41-
Verifier verifier = getVerifier("reproducible-build");
47+
public void testReproducible() throws Exception {
48+
verifier = getVerifier("reproducible-build");
4249
verifier.executeGoals(List.of("clean", "verify"));
4350
verifier.verifyErrorFreeLog();
4451

45-
// Check that the timestamp of the files inside the produced archives is equal
46-
// to the one specified in the "project.build.outputTimestamp" property of the
47-
// pom file.
52+
checkArchiveTimestamps();
53+
testBuildQualifier();
54+
testPropertiesFiles();
55+
}
56+
57+
/**
58+
* Checks that the timestamp of the files inside the produced archives is equal
59+
* to the one specified in the "project.build.outputTimestamp" property of the
60+
* pom file.
61+
*/
62+
private void checkArchiveTimestamps() throws Exception {
4863
checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0.jar");
4964
checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-attached.jar");
5065
checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar");
@@ -55,11 +70,8 @@ public void test() throws Exception {
5570
checkTimestamps(verifier.getBasedir() + "/reproducible.iu/target/reproducible.iu-1.0.0.zip");
5671
checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/reproducible.repository-1.0.0.zip");
5772
checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/p2-site.zip");
58-
59-
// Check that the build qualifier uses the timestamp specified in the
60-
// "project.build.outputTimestamp" property of the pom file.
61-
checkBuildQualifier(verifier.getBasedir()
62-
+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar");
73+
checkTimestamps(
74+
verifier.getBasedir() + "/reproducible.repository/target/products/main.product.id-linux.gtk.x86.zip");
6375
}
6476

6577
private void checkTimestamps(String file) throws IOException {
@@ -72,11 +84,33 @@ private void checkTimestamps(String file) throws IOException {
7284
}
7385
}
7486

75-
private void checkBuildQualifier(String file) throws IOException {
87+
/**
88+
* Checks that the build qualifier uses the timestamp specified in the
89+
* "project.build.outputTimestamp" property of the pom file.
90+
*
91+
* @throws IOException
92+
*/
93+
private void testBuildQualifier() throws IOException {
94+
final String file = verifier.getBasedir()
95+
+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar";
7696
try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
77-
Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
78-
List<String> lines = Files.readAllLines(manifest);
97+
final Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
98+
final List<String> lines = Files.readAllLines(manifest);
7999
Assert.assertTrue(lines.stream().anyMatch(l -> l.equals("Bundle-Version: 1.0.0.202301010000")));
80100
}
81101
}
102+
103+
/**
104+
* Checks that the generated properties files are reproducible.
105+
*
106+
* @throws IOException
107+
*/
108+
private void testPropertiesFiles() throws IOException {
109+
final String file = verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar";
110+
try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
111+
final Path propFile = fileSystem.getPath("OSGI-INF/l10n/bundle-src.properties");
112+
final String content = Files.readString(propFile, StandardCharsets.ISO_8859_1);
113+
Assert.assertEquals("bundleName=Reproducible-bundle Source\n" + "bundleVendor=unknown\n", content);
114+
}
115+
}
82116
}

‎tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/MavenP2SiteMojo.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
import org.eclipse.equinox.p2.core.IProvisioningAgent;
7373
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
7474
import org.eclipse.tycho.PackagingType;
75+
import org.eclipse.tycho.ReproducibleUtils;
7576
import org.eclipse.tycho.TychoConstants;
7677
import org.eclipse.tycho.core.PGPService;
7778
import org.eclipse.tycho.p2maven.tools.TychoFeaturesAndBundlesPublisherApplication;
@@ -484,9 +485,7 @@ protected File createMavenAdvice(Artifact artifact) throws MojoExecutionExceptio
484485
addProvidesAndProperty(properties, TychoConstants.PROP_EXTENSION, artifact.getType(), cnt++);
485486
addProvidesAndProperty(properties, TychoConstants.PROP_CLASSIFIER, artifact.getClassifier(), cnt++);
486487
addProvidesAndProperty(properties, "maven-scope", artifact.getScope(), cnt++);
487-
try (OutputStream os = new BufferedOutputStream(new FileOutputStream(p2))) {
488-
properties.store(os, null);
489-
}
488+
ReproducibleUtils.storeProperties(properties, p2.toPath());
490489
return p2;
491490
} catch (IOException e) {
492491
throw new MojoExecutionException("failed to generate p2.inf", e);

‎tycho-packaging-plugin/src/main/java/org/eclipse/tycho/packaging/reverseresolve/MavenCentralArtifactCoordinateResolver.java

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,10 @@
1212
*******************************************************************************/
1313
package org.eclipse.tycho.packaging.reverseresolve;
1414

15-
import java.io.BufferedOutputStream;
1615
import java.io.File;
1716
import java.io.FileInputStream;
18-
import java.io.FileOutputStream;
1917
import java.io.IOException;
2018
import java.io.InputStream;
21-
import java.io.OutputStream;
2219
import java.nio.file.Files;
2320
import java.security.MessageDigest;
2421
import java.util.List;
@@ -36,6 +33,7 @@
3633
import org.codehaus.plexus.component.annotations.Component;
3734
import org.codehaus.plexus.component.annotations.Requirement;
3835
import org.codehaus.plexus.logging.Logger;
36+
import org.eclipse.tycho.ReproducibleUtils;
3937
import org.eclipse.tycho.core.shared.MavenContext;
4038
import org.eclipse.tycho.p2.repository.GAV;
4139
import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -160,9 +158,7 @@ private void cacheResult(File cacheFile, Dependency dependency) {
160158
properties.setProperty(KEY_ARTIFACT_ID, dependency.getArtifactId());
161159
properties.setProperty(KEY_VERSION, dependency.getVersion());
162160
properties.setProperty(KEY_TYPE, dependency.getType());
163-
try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(cacheFile))) {
164-
properties.store(stream, null);
165-
}
161+
ReproducibleUtils.storeProperties(properties, cacheFile.toPath());
166162
} catch (IOException e) {
167163
// can't create cache file then...
168164
}

0 commit comments

Comments
 (0)