diff --git a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ReplicationCookie.java b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ReplicationCookie.java new file mode 100644 index 000000000..dbf40427f --- /dev/null +++ b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ReplicationCookie.java @@ -0,0 +1,66 @@ +// This software is released into the Public Domain. See copying.txt for details. +package org.openstreetmap.osmosis.replication.common; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.openstreetmap.osmosis.core.OsmosisRuntimeException; + +/** + * Cookie to be sent with all HTTP requests. The cookie is read from a file. + * + * @author Michael Reichert + */ +public class ReplicationCookie { + private static final String COOKIE_FILE_NAME = "cookie.txt"; + + private Path directory; + private String data; + + /** + * Creates an empty cookie. + * + * @param cookie_directory directory to read the cookie.txt from + */ + public ReplicationCookie(Path cookie_directory) { + directory = cookie_directory; + data = ""; + } + + /** + * Check if this cookie is not empty and used. + * + * @return False if it has not been set. + */ + public boolean valid() { + return !data.isEmpty(); + } + + /** + * Get the string representation of the cookie to be set as HTTP header. + */ + public String toString() { + return data; + } + + /** + * Read the cookie from a file name cookie.txt in the working directory. + */ + public void read() { + Path cookieFilePath = directory.resolve(Paths.get(COOKIE_FILE_NAME)); + try { + List lines = Files.readAllLines(cookieFilePath, Charset.forName("US-ASCII")); + if (lines.size() == 1) { + data = lines.get(0); + } else { + throw new OsmosisRuntimeException("The cookie file " + cookieFilePath.toString() + " must contain exactly one line."); + } + } catch (IOException e) { + throw new OsmosisRuntimeException("Failed to read the cookie file " + cookieFilePath.toString()); + } + } +} diff --git a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ServerStateReader.java b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ServerStateReader.java index 175ebe57a..644c02bc7 100644 --- a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ServerStateReader.java +++ b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ServerStateReader.java @@ -10,9 +10,11 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.logging.Logger; import java.util.Properties; import org.openstreetmap.osmosis.core.OsmosisRuntimeException; +import org.openstreetmap.osmosis.replication.v0_6.BaseReplicationDownloader; import org.openstreetmap.osmosis.core.OsmosisConstants; @@ -38,18 +40,36 @@ public ServerStateReader() { /** * Retrieves the latest state from the server. * + * No cookie will be sent with the HTTP request. + * * @param baseUrl * The url of the directory containing change files. * @return The state. */ public ReplicationState getServerState(URL baseUrl) { - return getServerState(baseUrl, SERVER_STATE_FILE); + return getServerState(baseUrl, SERVER_STATE_FILE, new ReplicationCookie()); + } + + + /** + * Retrieves the latest state from the server. + * + * @param baseUrl + * The url of the directory containing change files. + * @param cookie + * Cookie to send with each HTTP request. + * @return The state. + */ + public ReplicationState getServerState(URL baseUrl, ReplicationCookie cookie) { + return getServerState(baseUrl, SERVER_STATE_FILE, cookie); } /** * Retrieves the specified state from the server. * + * No cookie will be sent with the HTTP request. + * * @param baseUrl * The url of the directory containing change files. * @param sequenceNumber @@ -57,7 +77,23 @@ public ReplicationState getServerState(URL baseUrl) { * @return The state. */ public ReplicationState getServerState(URL baseUrl, long sequenceNumber) { - return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX)); + return getServerState(baseUrl, sequenceNumber, new ReplicationCookie()); + } + + + /** + * Retrieves the specified state from the server. + * + * @param baseUrl + * The url of the directory containing change files. + * @param sequenceNumber + * The sequence number of the state to be retrieved from the server. + * @param cookie + * Cookie to send with each HTTP request. + * @return The state. + */ + public ReplicationState getServerState(URL baseUrl, long sequenceNumber, ReplicationCookie cookie) { + return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX), cookie); } @@ -68,9 +104,11 @@ public ReplicationState getServerState(URL baseUrl, long sequenceNumber) { * The url of the directory containing change files. * @param stateFile * The state file to be retrieved. + * @param cookie + * Cookie to send with each HTTP request. * @return The state. */ - private ReplicationState getServerState(URL baseUrl, String stateFile) { + private ReplicationState getServerState(URL baseUrl, String stateFile, ReplicationCookie cookie) { URL stateUrl; try { @@ -88,6 +126,9 @@ private ReplicationState getServerState(URL baseUrl, String stateFile) { connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION); + if (cookie.valid()) { + connection.setRequestProperty("Cookie", cookie.toString()); + } try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { stateProperties = new Properties(); stateProperties.load(reader); diff --git a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/BaseReplicationDownloader.java b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/BaseReplicationDownloader.java index 6f46093a8..b03bb3a2d 100644 --- a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/BaseReplicationDownloader.java +++ b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/BaseReplicationDownloader.java @@ -9,8 +9,13 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -19,6 +24,7 @@ import org.openstreetmap.osmosis.core.task.common.RunnableTask; import org.openstreetmap.osmosis.core.util.FileBasedLock; import org.openstreetmap.osmosis.core.util.PropertiesPersister; +import org.openstreetmap.osmosis.replication.common.ReplicationCookie; import org.openstreetmap.osmosis.replication.common.ReplicationSequenceFormatter; import org.openstreetmap.osmosis.replication.common.ReplicationState; import org.openstreetmap.osmosis.replication.common.ServerStateReader; @@ -45,6 +51,8 @@ public abstract class BaseReplicationDownloader implements RunnableTask { private File workingDirectory; private ReplicationSequenceFormatter sequenceFormatter; private ServerStateReader serverStateReader; + private ReplicationDownloaderConfiguration configuration; + private ReplicationCookie cookie; /** @@ -58,6 +66,12 @@ public BaseReplicationDownloader(File workingDirectory) { sequenceFormatter = new ReplicationSequenceFormatter(9, 3); serverStateReader = new ServerStateReader(); + + configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE)); + cookie = new ReplicationCookie(workingDirectory.toPath()); + if (configuration.getAttachCookie()) { + cookie.read(); + } } @@ -98,7 +112,10 @@ private File downloadReplicationFile(String fileName, URL baseUrl) { connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION); - + if (cookie.valid()) { + connection.setRequestProperty("Cookie", cookie.toString()); + } + try (BufferedInputStream source = new BufferedInputStream(connection.getInputStream(), 65536)) { // Create a temporary file to write the data to. outputFile = File.createTempFile("change", null); @@ -174,8 +191,7 @@ protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration conf } - private ReplicationState download(ReplicationDownloaderConfiguration configuration, ReplicationState serverState, - ReplicationState initialLocalState) { + private ReplicationState download(ReplicationState serverState, ReplicationState initialLocalState) { URL baseUrl; ReplicationState localState; Date maximumDownloadTimestamp; @@ -210,7 +226,7 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati LOG.finer("Processing replication sequence " + sequenceNumber + "."); // Get the state associated with the next file. - fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber); + fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber, cookie); // Ensure that the next state is within the allowable timestamp // range. We must stop if the next data takes us beyond the maximum @@ -244,17 +260,13 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati private void runImpl() { try { - ReplicationDownloaderConfiguration configuration; ReplicationState serverState; ReplicationState localState; PropertiesPersister localStatePersistor; - // Instantiate utility objects. - configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE)); - // Obtain the server state. LOG.fine("Reading current server state."); - serverState = serverStateReader.getServerState(configuration.getBaseUrl()); + serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie); // Build the local state persister which is used for both loading and storing local state. localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE)); @@ -268,7 +280,7 @@ private void runImpl() { localState = new ReplicationState(localStatePersistor.loadMap()); // Download and process the replication files. - localState = download(configuration, serverState, localState); + localState = download(serverState, localState); } else { localState = serverState; diff --git a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java index 0e6f81a50..53fcc1126 100644 --- a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java +++ b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java @@ -9,6 +9,7 @@ import org.openstreetmap.osmosis.core.task.common.RunnableTask; import org.openstreetmap.osmosis.core.util.FileBasedLock; import org.openstreetmap.osmosis.core.util.PropertiesPersister; +import org.openstreetmap.osmosis.replication.common.ReplicationCookie; import org.openstreetmap.osmosis.replication.common.ReplicationState; import org.openstreetmap.osmosis.replication.common.ServerStateReader; import org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationDownloaderConfiguration; @@ -59,10 +60,16 @@ private void getLag() { // Instantiate utility objects. configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE)); + + // read cookie if necessary + ReplicationCookie cookie = new ReplicationCookie(workingDirectory.toPath()); + if (configuration.getAttachCookie()) { + cookie.read(); + } // Obtain the server state. LOG.fine("Reading current server state."); - serverState = serverStateReader.getServerState(configuration.getBaseUrl()); + serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie); // Build the local state persister which is used for both loading and storing local state. localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE)); diff --git a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/impl/ReplicationDownloaderConfiguration.java b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/impl/ReplicationDownloaderConfiguration.java index 19465f070..6f6f08923 100644 --- a/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/impl/ReplicationDownloaderConfiguration.java +++ b/osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/impl/ReplicationDownloaderConfiguration.java @@ -18,6 +18,7 @@ public class ReplicationDownloaderConfiguration { private static final String KEY_BASE_URL = "baseUrl"; private static final String KEY_MAX_INTERVAL = "maxInterval"; + private static final String ATTACH_COOKIE = "attachCookie"; private Properties properties; @@ -65,4 +66,13 @@ public URL getBaseUrl() { public int getMaxInterval() { return Integer.parseInt(properties.getProperty(KEY_MAX_INTERVAL)) * 1000; } + + /** + * Returns whether a cookie stored in cookie.txt should be sent with each request. + * + * @return If a cookie should be send. + */ + public boolean getAttachCookie() { + return Boolean.parseBoolean(properties.getProperty(ATTACH_COOKIE)); + } }