Skip to content

Commit

Permalink
Allow cookies to be sent with HTTP replication requests
Browse files Browse the repository at this point in the history
  • Loading branch information
Nakaner committed May 8, 2018
1 parent 2219470 commit 7d7ead3
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 22 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.replication.common;

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;

import org.openstreetmap.osmosis.core.OsmosisRuntimeException;

/**
* Cookie to be sent with all HTTP requests. The cookie is read from a file.
*
* @author Michael Reichert
*/
public class ReplicationCookie {
private static final String COOKIE_FILE_NAME = "cookie.txt";

private Path directory;
private String data;

/**
* Create an invalid dummy cookie.
*/
public ReplicationCookie() {
directory = null;
data = null;
}

/**
* Creates an empty cookie.
*
* @param cookieDirectory directory to read the cookie.txt from
*/
public ReplicationCookie(Path cookieDirectory) {
directory = cookieDirectory;
data = "";
}

/**
* Check if this cookie is not empty and used.
*
* @return False if it has not been set.
*/
public boolean valid() {
return data != null && !data.isEmpty();
}

/**
* Get the string representation of the cookie to be set as HTTP header.
*
* @return string representation
*/
public String toString() {
return data;
}

/**
* Read the cookie from a file name cookie.txt in the working directory.
*/
public void read() {
Path cookieFilePath = directory.resolve(Paths.get(COOKIE_FILE_NAME));
try {
List<String> lines = Files.readAllLines(cookieFilePath, Charset.forName("US-ASCII"));
if (lines.size() == 1) {
data = lines.get(0);
} else {
throw new OsmosisRuntimeException("The cookie file " + cookieFilePath.toString()
+ " must contain exactly one line.");
}
} catch (IOException e) {
throw new OsmosisRuntimeException("Failed to read the cookie file " + cookieFilePath.toString());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,60 @@ public ServerStateReader() {
/**
* Retrieves the latest state from the server.
*
* No cookie will be sent with the HTTP request.
*
* @param baseUrl
* The url of the directory containing change files.
* @return The state.
*/
public ReplicationState getServerState(URL baseUrl) {
return getServerState(baseUrl, SERVER_STATE_FILE);
return getServerState(baseUrl, SERVER_STATE_FILE, new ReplicationCookie());
}


/**
* Retrieves the latest state from the server.
*
* @param baseUrl
* The url of the directory containing change files.
* @param cookie
* Cookie to send with each HTTP request.
* @return The state.
*/
public ReplicationState getServerState(URL baseUrl, ReplicationCookie cookie) {
return getServerState(baseUrl, SERVER_STATE_FILE, cookie);
}


/**
* Retrieves the specified state from the server.
*
* No cookie will be sent with the HTTP request.
*
* @param baseUrl
* The url of the directory containing change files.
* @param sequenceNumber
* The sequence number of the state to be retrieved from the server.
* @return The state.
*/
public ReplicationState getServerState(URL baseUrl, long sequenceNumber) {
return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX));
return getServerState(baseUrl, sequenceNumber, new ReplicationCookie());
}


/**
* Retrieves the specified state from the server.
*
* @param baseUrl
* The url of the directory containing change files.
* @param sequenceNumber
* The sequence number of the state to be retrieved from the server.
* @param cookie
* Cookie to send with each HTTP request.
* @return The state.
*/
public ReplicationState getServerState(URL baseUrl, long sequenceNumber, ReplicationCookie cookie) {
return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX), cookie);
}


Expand All @@ -68,9 +102,11 @@ public ReplicationState getServerState(URL baseUrl, long sequenceNumber) {
* The url of the directory containing change files.
* @param stateFile
* The state file to be retrieved.
* @param cookie
* Cookie to send with each HTTP request.
* @return The state.
*/
private ReplicationState getServerState(URL baseUrl, String stateFile) {
private ReplicationState getServerState(URL baseUrl, String stateFile, ReplicationCookie cookie) {
URL stateUrl;

try {
Expand All @@ -88,6 +124,9 @@ private ReplicationState getServerState(URL baseUrl, String stateFile) {
connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);
if (cookie.valid()) {
connection.setRequestProperty("Cookie", cookie.toString());
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
stateProperties = new Properties();
stateProperties.load(reader);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.openstreetmap.osmosis.core.task.common.RunnableTask;
import org.openstreetmap.osmosis.core.util.FileBasedLock;
import org.openstreetmap.osmosis.core.util.PropertiesPersister;
import org.openstreetmap.osmosis.replication.common.ReplicationCookie;
import org.openstreetmap.osmosis.replication.common.ReplicationSequenceFormatter;
import org.openstreetmap.osmosis.replication.common.ReplicationState;
import org.openstreetmap.osmosis.replication.common.ServerStateReader;
Expand All @@ -45,6 +46,8 @@ public abstract class BaseReplicationDownloader implements RunnableTask {
private File workingDirectory;
private ReplicationSequenceFormatter sequenceFormatter;
private ServerStateReader serverStateReader;
protected ReplicationDownloaderConfiguration configuration;
private ReplicationCookie cookie;


/**
Expand All @@ -58,6 +61,12 @@ public BaseReplicationDownloader(File workingDirectory) {

sequenceFormatter = new ReplicationSequenceFormatter(9, 3);
serverStateReader = new ServerStateReader();

configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
cookie = new ReplicationCookie(workingDirectory.toPath());
if (configuration.getAttachCookie()) {
cookie.read();
}
}


Expand Down Expand Up @@ -98,7 +107,10 @@ private File downloadReplicationFile(String fileName, URL baseUrl) {
connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);

if (cookie.valid()) {
connection.setRequestProperty("Cookie", cookie.toString());
}

try (BufferedInputStream source = new BufferedInputStream(connection.getInputStream(), 65536)) {
// Create a temporary file to write the data to.
outputFile = File.createTempFile("change", null);
Expand Down Expand Up @@ -146,16 +158,13 @@ private void processReplicationFile(File replicationFile, ReplicationState repli
* limit the maximum timestamp further if needed. A sub-class may never increase the maximum
* timestamp beyond that calculated by this method.
*
* @param configuration
* The configuration.
* @param serverTimestamp
* The timestamp of the latest data on the server.
* @param localTimestamp
* The timestamp of the most recently downloaded data.
* @return The maximum timestamp for this invocation.
*/
protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration configuration, Date serverTimestamp,
Date localTimestamp) {
protected Date calculateMaximumTimestamp(Date serverTimestamp, Date localTimestamp) {
Date maximumTimestamp;

maximumTimestamp = serverTimestamp;
Expand All @@ -174,8 +183,7 @@ protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration conf
}


private ReplicationState download(ReplicationDownloaderConfiguration configuration, ReplicationState serverState,
ReplicationState initialLocalState) {
private ReplicationState download(ReplicationState serverState, ReplicationState initialLocalState) {
URL baseUrl;
ReplicationState localState;
Date maximumDownloadTimestamp;
Expand All @@ -187,7 +195,7 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati

// Determine the maximum timestamp that can be downloaded.
maximumDownloadTimestamp =
calculateMaximumTimestamp(configuration, serverState.getTimestamp(), localState.getTimestamp());
calculateMaximumTimestamp(serverState.getTimestamp(), localState.getTimestamp());
LOG.fine("The maximum timestamp to be downloaded is " + maximumDownloadTimestamp + ".");

// Download all files and send their contents to the sink.
Expand All @@ -210,7 +218,7 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati
LOG.finer("Processing replication sequence " + sequenceNumber + ".");

// Get the state associated with the next file.
fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber);
fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber, cookie);

// Ensure that the next state is within the allowable timestamp
// range. We must stop if the next data takes us beyond the maximum
Expand Down Expand Up @@ -244,17 +252,13 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati

private void runImpl() {
try {
ReplicationDownloaderConfiguration configuration;
ReplicationState serverState;
ReplicationState localState;
PropertiesPersister localStatePersistor;

// Instantiate utility objects.
configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));

// Obtain the server state.
LOG.fine("Reading current server state.");
serverState = serverStateReader.getServerState(configuration.getBaseUrl());
serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie);

// Build the local state persister which is used for both loading and storing local state.
localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
Expand All @@ -268,7 +272,7 @@ private void runImpl() {
localState = new ReplicationState(localStatePersistor.loadMap());

// Download and process the replication files.
localState = download(configuration, serverState, localState);
localState = download(serverState, localState);

} else {
localState = serverState;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,15 @@ private Date alignDateToIntervalBoundary(Date requestedDate, long intervalLength
* {@inheritDoc}
*/
@Override
protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration configuration, Date serverTimestamp,
Date localTimestamp) {
protected Date calculateMaximumTimestamp(Date serverTimestamp, Date localTimestamp) {
Date maximumTimestamp;
long intervalLength;

// Read the current persisted state.
currentDataState = replicationStore.getCurrentState();

// Get the default maximum timestamp according to base calculations.
maximumTimestamp = super.calculateMaximumTimestamp(configuration, serverTimestamp, localTimestamp);
maximumTimestamp = super.calculateMaximumTimestamp(serverTimestamp, localTimestamp);

// Align the maximum timestamp to an interval boundary.
intervalLength = getConfiguration().getIntervalLength();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.openstreetmap.osmosis.core.task.common.RunnableTask;
import org.openstreetmap.osmosis.core.util.FileBasedLock;
import org.openstreetmap.osmosis.core.util.PropertiesPersister;
import org.openstreetmap.osmosis.replication.common.ReplicationCookie;
import org.openstreetmap.osmosis.replication.common.ReplicationState;
import org.openstreetmap.osmosis.replication.common.ServerStateReader;
import org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationDownloaderConfiguration;
Expand Down Expand Up @@ -59,10 +60,16 @@ private void getLag() {

// Instantiate utility objects.
configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));

// read cookie if necessary
ReplicationCookie cookie = new ReplicationCookie(workingDirectory.toPath());
if (configuration.getAttachCookie()) {
cookie.read();
}

// Obtain the server state.
LOG.fine("Reading current server state.");
serverState = serverStateReader.getServerState(configuration.getBaseUrl());
serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie);

// Build the local state persister which is used for both loading and storing local state.
localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
public class ReplicationDownloaderConfiguration {
private static final String KEY_BASE_URL = "baseUrl";
private static final String KEY_MAX_INTERVAL = "maxInterval";
private static final String ATTACH_COOKIE = "attachCookie";


private Properties properties;
Expand Down Expand Up @@ -65,4 +66,13 @@ public URL getBaseUrl() {
public int getMaxInterval() {
return Integer.parseInt(properties.getProperty(KEY_MAX_INTERVAL)) * 1000;
}

/**
* Returns whether a cookie stored in cookie.txt should be sent with each request.
*
* @return If a cookie should be send.
*/
public boolean getAttachCookie() {
return Boolean.parseBoolean(properties.getProperty(ATTACH_COOKIE));
}
}

0 comments on commit 7d7ead3

Please sign in to comment.