From 956f6de0acbf25e45b5f16d95c3ed397d760405f Mon Sep 17 00:00:00 2001 From: "arnett, stu" Date: Wed, 24 Jun 2015 16:41:06 -0500 Subject: [PATCH] v2.0.0 --- src/main/java/com/emc/rest/smart/Host.java | 111 ++++++-- .../com/emc/rest/smart/HostListProvider.java | 9 +- .../java/com/emc/rest/smart/HostVetoRule.java | 33 +++ .../java/com/emc/rest/smart/LoadBalancer.java | 77 ++++-- .../com/emc/rest/smart/PollingDaemon.java | 31 ++- .../emc/rest/smart/SmartClientFactory.java | 6 +- .../java/com/emc/rest/smart/SmartConfig.java | 83 +++++- .../java/com/emc/rest/smart/SmartFilter.java | 58 +---- .../rest/smart/ecs/EcsHostListProvider.java | 236 ++++++++++++++++++ .../com/emc/rest/smart/ecs/ListDataNode.java | 57 +++++ src/main/java/com/emc/rest/smart/ecs/Vdc.java | 84 +++++++ .../com/emc/rest/smart/ecs/package-info.java | 30 +++ .../java/com/emc/rest/smart/HostTest.java | 97 +++++++ .../com/emc/rest/smart/LoadBalancerTest.java | 49 +--- .../com/emc/rest/smart/SmartClientTest.java | 4 +- .../com/emc/rest/smart/TestHealthCheck.java | 156 ++++++++++++ .../smart/ecs/EcsHostListProviderTest.java | 138 ++++++++++ .../emc/rest/smart/ecs/ListDataNodeTest.java | 73 ++++++ .../java/com/emc/util/RequestSimulator.java | 108 ++++++++ 19 files changed, 1283 insertions(+), 157 deletions(-) create mode 100644 src/main/java/com/emc/rest/smart/HostVetoRule.java create mode 100644 src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java create mode 100644 src/main/java/com/emc/rest/smart/ecs/ListDataNode.java create mode 100644 src/main/java/com/emc/rest/smart/ecs/Vdc.java create mode 100644 src/main/java/com/emc/rest/smart/ecs/package-info.java create mode 100644 src/test/java/com/emc/rest/smart/HostTest.java create mode 100644 src/test/java/com/emc/rest/smart/TestHealthCheck.java create mode 100644 src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java create mode 100644 src/test/java/com/emc/rest/smart/ecs/ListDataNodeTest.java create mode 100644 src/test/java/com/emc/util/RequestSimulator.java diff --git a/src/main/java/com/emc/rest/smart/Host.java b/src/main/java/com/emc/rest/smart/Host.java index 0b40de2..e8dfcae 100644 --- a/src/main/java/com/emc/rest/smart/Host.java +++ b/src/main/java/com/emc/rest/smart/Host.java @@ -29,8 +29,8 @@ import org.apache.log4j.LogMF; import org.apache.log4j.Logger; +import java.util.ArrayDeque; import java.util.Date; -import java.util.LinkedList; import java.util.Queue; /** @@ -46,12 +46,13 @@ public class Host implements HostStats { private static final Logger l4j = Logger.getLogger(Host.class); - protected static final int DEFAULT_RESPONSE_WINDOW_SIZE = 20; - protected static final int DEFAULT_ERROR_COOL_DOWN_SECS = 30; + public static final int DEFAULT_RESPONSE_WINDOW_SIZE = 25; + public static final int DEFAULT_ERROR_COOL_DOWN_SECS = 10; private String name; - protected int responseWindowSize; - protected int errorCoolDownSecs; + private boolean healthy = true; + protected int responseWindowSize = DEFAULT_RESPONSE_WINDOW_SIZE; + protected int errorCoolDownSecs = DEFAULT_ERROR_COOL_DOWN_SECS; protected int openConnections; protected long lastConnectionTime; @@ -60,24 +61,13 @@ public class Host implements HostStats { protected long consecutiveErrors; protected long responseQueueAverage; - protected Queue responseQueue = new LinkedList(); + protected Queue responseQueue = new ArrayDeque(); /** - * Uses a default error cool down of 30 secs and a response window size of 20. + * @param name the host name or IP address of this host */ public Host(String name) { - this(name, DEFAULT_RESPONSE_WINDOW_SIZE, DEFAULT_ERROR_COOL_DOWN_SECS); - } - - /** - * @param name the host name or IP address of this host - * @param errorCoolDownSecs the cool down period for errors (number of seconds after an error when the host is - * considered normalized). compounded for multiple consecutive errors - */ - public Host(String name, int responseWindowSize, int errorCoolDownSecs) { this.name = name; - this.responseWindowSize = responseWindowSize; - this.errorCoolDownSecs = errorCoolDownSecs; } public synchronized void connectionOpened() { @@ -102,7 +92,7 @@ public synchronized void callComplete(long duration, boolean isError) { // log response time responseQueue.add(duration); - if (responseQueue.size() > responseWindowSize) + while (responseQueue.size() > responseWindowSize) responseQueue.remove(); // recalculate average @@ -120,17 +110,39 @@ public String getName() { return name; } - public synchronized long getResponseIndex() { - // error adjustment adjust the index up based on the number of consecutive errors - long errorAdjustment = consecutiveErrors * errorCoolDownSecs * 1000; + public boolean isHealthy() { + return healthy; + } + + public void setHealthy(boolean healthy) { + this.healthy = healthy; + } + + public long getResponseIndex() { + long currentTime = System.currentTimeMillis(); - // open connection adjustment adjusts the index up based on the number of open connections to the host - long openConnectionAdjustment = openConnections * errorCoolDownSecs; // cool down secs as ms instead + synchronized (this) { + // error adjustment adjust the index up based on the number of consecutive errors + long errorAdjustment = consecutiveErrors * errorCoolDownSecs * 1000; - // dormant adjustment adjusts the index down based on how long it's been since the host was last used - long msSinceLastUse = System.currentTimeMillis() - lastConnectionTime; + // open connection adjustment adjusts the index up based on the number of open connections to the host + long openConnectionAdjustment = openConnections * errorCoolDownSecs; // cool down secs as ms instead - return responseQueueAverage + errorAdjustment + openConnectionAdjustment - msSinceLastUse; + // dormant adjustment adjusts the index down based on how long it's been since the host was last used + long msSinceLastUse = currentTime - lastConnectionTime; + + return responseQueueAverage + errorAdjustment + openConnectionAdjustment - msSinceLastUse; + } + } + + /** + * Resets historical metrics. Use with care! + */ + public synchronized void resetStats() { + totalConnections = openConnections; + totalErrors = 0; + consecutiveErrors = 0; + responseQueueAverage = 0; } @Override @@ -158,9 +170,54 @@ public long getResponseQueueAverage() { return responseQueueAverage; } + public long getConsecutiveErrors() { + return consecutiveErrors; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Host)) return false; + + Host host = (Host) o; + + return getName().equals(host.getName()); + } + + @Override + public int hashCode() { + return getName().hashCode(); + } + @Override public String toString() { return String.format("%s{totalConnections=%d, totalErrors=%d, openConnections=%d, lastConnectionTime=%s, responseQueueAverage=%d}", name, totalConnections, totalErrors, openConnections, new Date(lastConnectionTime).toString(), responseQueueAverage); } + + public int getResponseWindowSize() { + return responseWindowSize; + } + + public void setResponseWindowSize(int responseWindowSize) { + this.responseWindowSize = responseWindowSize; + } + + public int getErrorCoolDownSecs() { + return errorCoolDownSecs; + } + + public void setErrorCoolDownSecs(int errorCoolDownSecs) { + this.errorCoolDownSecs = errorCoolDownSecs; + } + + public Host withResponseWindowSize(int responseWindowSize) { + setResponseWindowSize(responseWindowSize); + return this; + } + + public Host withErrorCoolDownSecs(int errorCoolDownSecs) { + setErrorCoolDownSecs(errorCoolDownSecs); + return this; + } } diff --git a/src/main/java/com/emc/rest/smart/HostListProvider.java b/src/main/java/com/emc/rest/smart/HostListProvider.java index ff72d8f..2f34080 100644 --- a/src/main/java/com/emc/rest/smart/HostListProvider.java +++ b/src/main/java/com/emc/rest/smart/HostListProvider.java @@ -29,5 +29,12 @@ import java.util.List; public interface HostListProvider { - public abstract List getHostList(); + List getHostList(); + + /** + * If this completes without throwing an exception, the host is considered healthy + * (host.setHealthy(true) is called). Otherwise, the host is considered unhealthy/down + * (host.setHealthy(false) is called). + */ + void runHealthCheck(Host host); } diff --git a/src/main/java/com/emc/rest/smart/HostVetoRule.java b/src/main/java/com/emc/rest/smart/HostVetoRule.java new file mode 100644 index 0000000..dc1a32c --- /dev/null +++ b/src/main/java/com/emc/rest/smart/HostVetoRule.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart; + +import java.util.Map; + +public interface HostVetoRule { + boolean shouldVeto(Host host, Map requestProperties); +} diff --git a/src/main/java/com/emc/rest/smart/LoadBalancer.java b/src/main/java/com/emc/rest/smart/LoadBalancer.java index 9b257ed..94e8532 100644 --- a/src/main/java/com/emc/rest/smart/LoadBalancer.java +++ b/src/main/java/com/emc/rest/smart/LoadBalancer.java @@ -26,27 +26,22 @@ */ package com.emc.rest.smart; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.*; public class LoadBalancer { - private final Queue hosts = new ConcurrentLinkedQueue(); + private final Queue hosts = new ArrayDeque(); + private List vetoRules; - public LoadBalancer(List initialHosts) { + public LoadBalancer(List initialHosts) { // seed the host map - for (String host : initialHosts) { - hosts.add(new Host(host)); - } + hosts.addAll(initialHosts); } /** * Returns the host with the lowest response index. */ - public Host getTopHost() { + public Host getTopHost(Map requestProperties) { Host topHost = null; long lowestIndex = Long.MAX_VALUE; @@ -54,6 +49,12 @@ public Host getTopHost() { synchronized (hosts) { for (Host host : hosts) { + // apply any veto rules + if (shouldVeto(host, requestProperties)) continue; + + // if the host is unhealthy/down, ignore it + if (!host.isHealthy()) continue; + // get response index for a host long hostIndex = host.getResponseIndex(); @@ -72,16 +73,41 @@ public Host getTopHost() { return topHost; } + protected boolean shouldVeto(Host host, Map requestProperties) { + if (vetoRules != null) { + for (HostVetoRule vetoRule : vetoRules) { + if (vetoRule.shouldVeto(host, requestProperties)) return true; + } + } + return false; + } + + /** + * Returns a list of all known hosts. This list is a clone; modification will not affect the load balancer + */ + public synchronized List getAllHosts() { + return new ArrayList(hosts); + } + /** * Returns stats for all active hosts in this load balancer */ - public HostStats[] getHostStats() { + public synchronized HostStats[] getHostStats() { return hosts.toArray(new HostStats[hosts.size()]); } - protected void updateHosts(List updatedHosts) throws Exception { + /** + * Resets connection metrics. Use with care! + */ + public void resetStats() { + for (Host host : getAllHosts()) { + host.resetStats(); + } + } + + protected void updateHosts(List updatedHosts) throws Exception { // don't modify the parameter - List hostList = new ArrayList(updatedHosts); + List hostList = new ArrayList(updatedHosts); synchronized (hosts) { // remove hosts from stored list that are not present in updated list @@ -90,10 +116,10 @@ protected void updateHosts(List updatedHosts) throws Exception { while (hostI.hasNext()) { Host host = hostI.next(); boolean stillThere = false; - Iterator hostListI = hostList.iterator(); + Iterator hostListI = hostList.iterator(); while (hostListI.hasNext()) { - String hostFromUpdate = hostListI.next(); - if (host.getName().equalsIgnoreCase(hostFromUpdate)) { + Host hostFromUpdate = hostListI.next(); + if (host.equals(hostFromUpdate)) { // this host is in both the stored list and the updated list stillThere = true; @@ -107,9 +133,22 @@ protected void updateHosts(List updatedHosts) throws Exception { } // what's left in the updated list are new hosts, so add them - for (String newHost : hostList) { - hosts.add(new Host(newHost)); + for (Host newHost : hostList) { + hosts.add(newHost); } } } + + public List getVetoRules() { + return vetoRules; + } + + public void setVetoRules(List vetoRules) { + this.vetoRules = vetoRules; + } + + public LoadBalancer withVetoRules(HostVetoRule... vetoRules) { + setVetoRules(Arrays.asList(vetoRules)); + return this; + } } diff --git a/src/main/java/com/emc/rest/smart/PollingDaemon.java b/src/main/java/com/emc/rest/smart/PollingDaemon.java index 6885618..a37944e 100644 --- a/src/main/java/com/emc/rest/smart/PollingDaemon.java +++ b/src/main/java/com/emc/rest/smart/PollingDaemon.java @@ -54,24 +54,43 @@ public void run() { LoadBalancer loadBalancer = smartConfig.getLoadBalancer(); HostListProvider hostListProvider = smartConfig.getHostListProvider(); - if (smartConfig.isDisablePolling()) { - l4j.info("host polling is disabled; not updating hosts"); + if (!smartConfig.isHostUpdateEnabled()) { + l4j.info("host update is disabled; not updating hosts"); } else if (hostListProvider == null) { l4j.info("no host list provider; not updating hosts"); } else { try { loadBalancer.updateHosts(hostListProvider.getHostList()); } catch (Throwable t) { - l4j.warn("Unable to enumerate servers", t); + l4j.warn("unable to enumerate servers", t); + } + } + + if (!smartConfig.isHealthCheckEnabled()) { + l4j.info("health check is disabled; not checking hosts"); + } else if (hostListProvider == null) { + l4j.info("no host list provider; not checking hosts"); + } else { + for (Host host : loadBalancer.getAllHosts()) { + try { + hostListProvider.runHealthCheck(host); + host.setHealthy(true); + LogMF.debug(l4j, "health check successful for {0}; host is marked healthy", host.getName()); + } catch (Throwable t) { + host.setHealthy(false); + l4j.warn("health check failed for " + host.getName() + "; host is marked unhealthy", t); + } } } long callTime = System.currentTimeMillis() - start; try { - LogMF.debug(l4j, "polling daemon finished; sleeping for {0} seconds..", smartConfig.getPollInterval()); - Thread.sleep(smartConfig.getPollInterval() * 1000 - callTime); + long sleepTime = smartConfig.getPollInterval() * 1000 - callTime; + if (sleepTime < 0) sleepTime = 0; + LogMF.debug(l4j, "polling daemon finished; will poll again in {0}ms..", sleepTime); + if (sleepTime > 0) Thread.sleep(sleepTime); } catch (InterruptedException e) { - l4j.warn("Interrupted while sleeping"); + l4j.warn("interrupted while sleeping", e); } } } diff --git a/src/main/java/com/emc/rest/smart/SmartClientFactory.java b/src/main/java/com/emc/rest/smart/SmartClientFactory.java index db9c1d8..d932832 100644 --- a/src/main/java/com/emc/rest/smart/SmartClientFactory.java +++ b/src/main/java/com/emc/rest/smart/SmartClientFactory.java @@ -48,7 +48,7 @@ public static Client createSmartClient(SmartConfig smartConfig, Client client = createStandardClient(smartConfig, clientHandler); // inject SmartFilter (this is the Jersey integration point of the load balancer) - client.addFilter(new SmartFilter(smartConfig.getLoadBalancer())); + client.addFilter(new SmartFilter(smartConfig)); // set up polling for updated host list (if polling is disabled in smartConfig or there's no host list provider, // nothing will happen) @@ -80,7 +80,7 @@ public static Client createStandardClient(SmartConfig smartConfig, // pass in jersey parameters from calling code (allows customization of client) for (String propName : smartConfig.getProperties().keySet()) { - clientConfig.getProperties().put(propName, smartConfig.property(propName)); + clientConfig.getProperties().put(propName, smartConfig.getProperty(propName)); } // replace sized writers with override writers to allow dynamic content-length (i.e. for transformations) @@ -98,8 +98,6 @@ public static Client createStandardClient(SmartConfig smartConfig, // build Jersey client Client client = new Client(clientHandler, clientConfig); - // TODO: do we need a custom retry handler? - return client; } diff --git a/src/main/java/com/emc/rest/smart/SmartConfig.java b/src/main/java/com/emc/rest/smart/SmartConfig.java index dca9716..f234577 100644 --- a/src/main/java/com/emc/rest/smart/SmartConfig.java +++ b/src/main/java/com/emc/rest/smart/SmartConfig.java @@ -27,6 +27,7 @@ package com.emc.rest.smart; import java.net.URI; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,21 +42,34 @@ public class SmartConfig { private String proxyUser; private String proxyPass; - private List initialHosts; private LoadBalancer loadBalancer; private HostListProvider hostListProvider; private int pollInterval = DEFAULT_POLL_INTERVAL; - private boolean disablePolling = false; + private boolean hostUpdateEnabled = true; + private boolean healthCheckEnabled = true; private Map properties = new HashMap(); - public SmartConfig(List initialHosts) { - this.initialHosts = initialHosts; - this.loadBalancer = new LoadBalancer(initialHosts); + /** + * @see #SmartConfig(LoadBalancer) + */ + public SmartConfig(String... initialHostNames) { + List hostList = new ArrayList(); + for (String hostName : initialHostNames) { + hostList.add(new Host(hostName)); + } + this.loadBalancer = new LoadBalancer(hostList); + } + + /** + * Constructs a new SmartConfig using a new {@link LoadBalancer} seeded with the specified hosts + */ + public SmartConfig(List initialHosts) { + this(new LoadBalancer(initialHosts)); } - public List getInitialHosts() { - return initialHosts; + public SmartConfig(LoadBalancer loadBalancer) { + this.loadBalancer = loadBalancer; } public synchronized LoadBalancer getLoadBalancer() { @@ -105,26 +119,69 @@ public void setPollInterval(int pollInterval) { this.pollInterval = pollInterval; } - public boolean isDisablePolling() { - return disablePolling; + public boolean isHostUpdateEnabled() { + return hostUpdateEnabled; } - public void setDisablePolling(boolean disablePolling) { - this.disablePolling = disablePolling; + public void setHostUpdateEnabled(boolean hostUpdateEnabled) { + this.hostUpdateEnabled = hostUpdateEnabled; + } + + public boolean isHealthCheckEnabled() { + return healthCheckEnabled; + } + + public void setHealthCheckEnabled(boolean healthCheckEnabled) { + this.healthCheckEnabled = healthCheckEnabled; } public Map getProperties() { return properties; } - public Object property(String propName) { + public Object getProperty(String propName) { return properties.get(propName); } /** * Allows custom Jersey client properties to be set. These will be passed on in the Jersey ClientConfig */ - public void property(String propName, Object value) { + public void withProperty(String propName, Object value) { properties.put(propName, value); } + + public SmartConfig withProxyUri(URI proxyUri) { + setProxyUri(proxyUri); + return this; + } + + public SmartConfig withProxyUser(String proxyUser) { + setProxyUser(proxyUser); + return this; + } + + public SmartConfig withProxyPass(String proxyPass) { + setProxyPass(proxyPass); + return this; + } + + public SmartConfig withHostListProvider(HostListProvider hostListProvider) { + setHostListProvider(hostListProvider); + return this; + } + + public SmartConfig withPollInterval(int pollInterval) { + setPollInterval(pollInterval); + return this; + } + + public SmartConfig withHostUpdateEnabled(boolean hostUpdateEnabled) { + setHostUpdateEnabled(hostUpdateEnabled); + return this; + } + + public SmartConfig withHealthCheckEnabled(boolean healthCheckEnabled) { + setHealthCheckEnabled(healthCheckEnabled); + return this; + } } diff --git a/src/main/java/com/emc/rest/smart/SmartFilter.java b/src/main/java/com/emc/rest/smart/SmartFilter.java index ca2fcd4..96a4f04 100644 --- a/src/main/java/com/emc/rest/smart/SmartFilter.java +++ b/src/main/java/com/emc/rest/smart/SmartFilter.java @@ -31,6 +31,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.filter.ClientFilter; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -39,10 +40,10 @@ public class SmartFilter extends ClientFilter { public static final String BYPASS_LOAD_BALANCER = "com.emc.rest.smart.bypassLoadBalancer"; - private LoadBalancer loadBalancer; + private SmartConfig smartConfig; - public SmartFilter(LoadBalancer loadBalancer) { - this.loadBalancer = loadBalancer; + public SmartFilter(SmartConfig smartConfig) { + this.smartConfig = smartConfig; } @Override @@ -54,7 +55,7 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio } // get highest ranked host for next request - Host host = loadBalancer.getTopHost(); + Host host = smartConfig.getLoadBalancer().getTopHost(request.getProperties()); // replace the host in the request URI uri = request.getURI(); @@ -93,41 +94,15 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio /** * captures closure in host statistics */ - protected class WrappedInputStream extends InputStream { - private InputStream delegated; + protected class WrappedInputStream extends FilterInputStream { private Host host; private boolean closed = false; - public WrappedInputStream(InputStream delegated, Host host) { - this.delegated = delegated; + public WrappedInputStream(InputStream in, Host host) { + super(in); this.host = host; } - @Override - public int read() throws IOException { - return delegated.read(); - } - - @Override - public int read(byte[] b) throws IOException { - return delegated.read(b); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return delegated.read(b, off, len); - } - - @Override - public long skip(long n) throws IOException { - return delegated.skip(n); - } - - @Override - public int available() throws IOException { - return delegated.available(); - } - @Override public void close() throws IOException { synchronized (this) { @@ -136,22 +111,7 @@ public void close() throws IOException { closed = true; } } - delegated.close(); - } - - @Override - public void mark(int readlimit) { - delegated.mark(readlimit); - } - - @Override - public void reset() throws IOException { - delegated.reset(); - } - - @Override - public boolean markSupported() { - return delegated.markSupported(); + super.close(); } } } diff --git a/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java new file mode 100644 index 0000000..bb0f33b --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import com.emc.rest.smart.Host; +import com.emc.rest.smart.HostListProvider; +import com.emc.rest.smart.LoadBalancer; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.LogMF; +import org.apache.log4j.Logger; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.net.URI; +import java.net.URISyntaxException; +import java.text.SimpleDateFormat; +import java.util.*; + +public class EcsHostListProvider implements HostListProvider { + private static final Logger l4j = Logger.getLogger(EcsHostListProvider.class); + + public static final String DEFAULT_PROTOCOL = "https"; + public static final int DEFAULT_PORT = 9021; + + protected final SimpleDateFormat rfc822DateFormat; + private Client client; + private LoadBalancer loadBalancer; + private String user; + private String secret; + private String protocol = DEFAULT_PROTOCOL; + private int port = DEFAULT_PORT; + private List vdcs; + + public EcsHostListProvider(Client client, LoadBalancer loadBalancer, String user, String secret) { + this.client = client; + this.loadBalancer = loadBalancer; + this.user = user; + this.secret = secret; + rfc822DateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); + rfc822DateFormat.setTimeZone(new SimpleTimeZone(0, "GMT")); + } + + public List getHostList() { + if (vdcs == null || vdcs.isEmpty()) return getDataNodes(loadBalancer.getTopHost(null)); + + List hostList = new ArrayList(); + + for (Vdc vdc : vdcs) { + if (vdc.getHosts().isEmpty()) l4j.warn("VDC " + vdc.getName() + " has no hosts!"); + + boolean success = false; + for (Host host : vdc) { + if (!host.isHealthy()) { // the load balancer manages health checks + l4j.warn("not retrieving node list from " + host.getName() + " because it's unhealthy"); + continue; + } + try { + updateVdcNodes(vdc, getDataNodes(host)); + success = true; + break; + } catch (Throwable t) { + l4j.warn("unable to retrieve node list from " + host.getName(), t); + } + } + if (!success) l4j.warn("could not retrieve node list for VDC " + vdc.getName()); + + hostList.addAll(vdc.getHosts()); + } + + return hostList; + } + + @Override + public void runHealthCheck(Host host) { + // header is workaround for STORAGE-1833 + client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x").get(String.class); + } + + protected List getDataNodes(Host host) { + String path = "/?endpoint"; + URI uri = getRequestUri(host, path); + + // format date + String rfcDate; + synchronized (rfc822DateFormat) { + rfcDate = rfc822DateFormat.format(new Date()); + } + + // generate signature + String canonicalString = "GET\n\n\n" + rfcDate + "\n" + path; + String signature; + try { + signature = getSignature(canonicalString, secret); + } catch (Exception e) { + throw new RuntimeException("could not generate signature", e); + } + + // construct request + WebResource.Builder request = client.resource(uri).getRequestBuilder(); + + // add date and auth headers + request.header("Date", rfcDate); + request.header("Authorization", "AWS " + user + ":" + signature); + + // make REST call + LogMF.debug(l4j, "retrieving VDC node list from {0}", host.getName()); + List dataNodes = request.get(ListDataNode.class).getDataNodes(); + + List hosts = new ArrayList(); + for (String node : dataNodes) { + hosts.add(new Host(node)); + } + return hosts; + } + + protected URI getRequestUri(Host host, String path) { + try { + String portStr = (port > -1) ? ":" + port : ""; + return new URI(protocol + "://" + host.getName() + portStr + path); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + protected String getSignature(String canonicalString, String secret) throws Exception { + Mac mac = Mac.getInstance("HmacSHA1"); + mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA1")); + String signature = new String(Base64.encodeBase64(mac.doFinal(canonicalString.getBytes("UTF-8")))); + l4j.debug("canonicalString:\n" + canonicalString); + l4j.debug("signature:\n" + signature); + return signature; + } + + protected void updateVdcNodes(Vdc vdc, List nodeList) { + if (nodeList == null || nodeList.isEmpty()) throw new RuntimeException("node list is empty"); + + // we need to maintain references to existing hosts to preserve health status, which is managed by the load + // balancer + for (Iterator vdcI = vdc.iterator(); vdcI.hasNext(); ) { + Host vdcHost = vdcI.next(); + boolean hostPresent = false; + for (Iterator nodeI = nodeList.iterator(); nodeI.hasNext(); ) { + Host node = nodeI.next(); + + // already aware of this node; remove from new node list + if (vdcHost.equals(node)) { + hostPresent = true; + nodeI.remove(); + } + } + + // host is not in the updated host list, so remove it from the VDC + if (!hostPresent) { + l4j.info("host " + vdcHost.getName() + " was not in the updated node list; removing from VDC " + vdc.getName()); + vdcI.remove(); + } + } + + // add any remaining new hosts we weren't previously aware of + for (Host node : nodeList) { + l4j.info("adding host " + node.getName() + " to VDC " + vdc.getName()); + vdc.getHosts().add(node); + } + } + + public Client getClient() { + return client; + } + + public LoadBalancer getLoadBalancer() { + return loadBalancer; + } + + public String getUser() { + return user; + } + + public String getSecret() { + return secret; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public List getVdcs() { + return vdcs; + } + + public void setVdcs(List vdcs) { + this.vdcs = vdcs; + } + + public EcsHostListProvider withVdcs(Vdc... vdcs) { + setVdcs(Arrays.asList(vdcs)); + return this; + } +} diff --git a/src/main/java/com/emc/rest/smart/ecs/ListDataNode.java b/src/main/java/com/emc/rest/smart/ecs/ListDataNode.java new file mode 100644 index 0000000..083babb --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/ListDataNode.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElements; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +@XmlRootElement(name = "ListDataNode") +public class ListDataNode { + private List dataNodes = new ArrayList(); + private String versionInfo; + + @XmlElements(@XmlElement(name = "DataNodes")) + public List getDataNodes() { + return dataNodes; + } + + public void setDataNodes(List dataNodes) { + this.dataNodes = dataNodes; + } + + @XmlElement(name = "VersionInfo") + public String getVersionInfo() { + return versionInfo; + } + + public void setVersionInfo(String versionInfo) { + this.versionInfo = versionInfo; + } +} diff --git a/src/main/java/com/emc/rest/smart/ecs/Vdc.java b/src/main/java/com/emc/rest/smart/ecs/Vdc.java new file mode 100644 index 0000000..d6213fc --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/Vdc.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import com.emc.rest.smart.Host; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class Vdc implements Iterable { + private String name; + private List hosts; + + public Vdc(String... hostNames) { + this.name = hostNames[0]; + hosts = new ArrayList(); + for (String hostName : hostNames) { + hosts.add(new Host(hostName)); + } + } + + public Vdc(List hosts) { + this(hosts.get(0).getName(), hosts); + } + + public Vdc(String name, List hosts) { + this.name = name; + this.hosts = hosts; + } + + @Override + public Iterator iterator() { + return hosts.iterator(); + } + + public boolean isHealthy() { + for (Host host : hosts) { + if (!host.isHealthy()) return false; + } + return true; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getHosts() { + return hosts; + } + + public Vdc withName(String name) { + setName(name); + return this; + } +} diff --git a/src/main/java/com/emc/rest/smart/ecs/package-info.java b/src/main/java/com/emc/rest/smart/ecs/package-info.java new file mode 100644 index 0000000..13cddb1 --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/package-info.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +@XmlSchema(namespace = "http://s3.amazonaws.com/doc/2006-03-01/", elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED) +package com.emc.rest.smart.ecs; + +import javax.xml.bind.annotation.XmlSchema; \ No newline at end of file diff --git a/src/test/java/com/emc/rest/smart/HostTest.java b/src/test/java/com/emc/rest/smart/HostTest.java new file mode 100644 index 0000000..b6838b6 --- /dev/null +++ b/src/test/java/com/emc/rest/smart/HostTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart; + +import org.junit.Assert; +import org.junit.Test; + +public class HostTest { + @Test + public void testHost() throws Exception { + Host host = new Host("foo"); + int errorCoolDown = host.getErrorCoolDownSecs(); + + // simulate some calls + int callCount = 100, duration = 50; + for (int i = 0; i < callCount; i++) { + host.connectionOpened(); + host.callComplete(duration, false); + host.connectionClosed(); + } + + Assert.assertEquals(callCount, host.getTotalConnections()); + Assert.assertEquals(0, host.getTotalErrors()); + Assert.assertEquals(0, host.getConsecutiveErrors()); + Assert.assertEquals(0, host.getOpenConnections()); + Assert.assertEquals(duration, host.getResponseQueueAverage()); + Assert.assertTrue(duration - host.getResponseIndex() <= 1); // 1ms margin of error (execution time) + + // test open connections + host.connectionOpened(); + host.connectionOpened(); + + Assert.assertEquals(2, host.getOpenConnections()); + + // test error + host.callComplete(duration, true); + host.connectionClosed(); + + Assert.assertEquals(1, host.getOpenConnections()); + Assert.assertEquals(1, host.getConsecutiveErrors()); + Assert.assertEquals(1, host.getTotalErrors()); + Assert.assertEquals(duration, host.getResponseQueueAverage()); + // response average + (errors * error-cool-down-in-secs) + (open-connections * error-cool-down-in-ms) + Assert.assertTrue(host.getResponseIndex() - (50 + errorCoolDown * 1000 + errorCoolDown) <= 1); // 1ms margin of error (execution time) + + // test another error + host.callComplete(duration, true); + host.connectionClosed(); + + Assert.assertEquals(0, host.getOpenConnections()); + Assert.assertEquals(2, host.getConsecutiveErrors()); + Assert.assertEquals(2, host.getTotalErrors()); + Assert.assertEquals(duration, host.getResponseQueueAverage()); + // response average + (errors * error-cool-down-in-secs) + (open-connections * error-cool-down-in-ms) + Assert.assertTrue(host.getResponseIndex() - (50 + 2 * errorCoolDown * 1000) <= 1); // 1ms margin of error (execution time) + + // test cool-down + Thread.sleep(500); + + Assert.assertTrue(host.getResponseIndex() - (50 + 2 * errorCoolDown * 1000 - 500) <= 1); // 1ms margin of error (execution time) + + // test no more errors + host.connectionOpened(); + host.callComplete(duration, false); + host.connectionClosed(); + + Assert.assertEquals(0, host.getConsecutiveErrors()); + Assert.assertEquals(2, host.getTotalErrors()); + Assert.assertEquals(callCount + 3, host.getTotalConnections()); + Assert.assertEquals(duration, host.getResponseQueueAverage()); + Assert.assertTrue(host.getResponseIndex() - 50 <= 1); // 1ms margin of error (execution time) + } +} diff --git a/src/test/java/com/emc/rest/smart/LoadBalancerTest.java b/src/test/java/com/emc/rest/smart/LoadBalancerTest.java index edeabc6..fe67474 100644 --- a/src/test/java/com/emc/rest/smart/LoadBalancerTest.java +++ b/src/test/java/com/emc/rest/smart/LoadBalancerTest.java @@ -26,6 +26,7 @@ */ package com.emc.rest.smart; +import com.emc.util.RequestSimulator; import org.apache.log4j.Level; import org.apache.log4j.LogMF; import org.apache.log4j.Logger; @@ -35,7 +36,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -48,41 +48,16 @@ public class LoadBalancerTest { public void testDistribution() throws Exception { String[] hostList = new String[]{"foo", "bar", "baz", "biz"}; final int callCount = 1000, callDuration = 50; - final Random random = new Random(); - SmartConfig smartConfig = new SmartConfig(Arrays.asList(hostList)); + SmartConfig smartConfig = new SmartConfig(hostList); smartConfig.setPollInterval(1); final LoadBalancer loadBalancer = smartConfig.getLoadBalancer(); - // simulate callCount successful calls with identical response times - ExecutorService executorService = Executors.newFixedThreadPool(10); - List futures = new ArrayList(); - for (int i = 0; i < callCount; i++) { - futures.add(executorService.submit(new Runnable() { - @Override - public void run() { - int waitMs; - synchronized (random) { - waitMs = random.nextInt(20); - } - try { - Thread.sleep(waitMs); - } catch (InterruptedException e) { - l4j.warn("thread interrupted", e); - } - Host host = loadBalancer.getTopHost(); - host.connectionOpened(); - host.callComplete(callDuration, false); - host.connectionClosed(); - } - })); - } + RequestSimulator simulator = new RequestSimulator(loadBalancer, callCount, callDuration); + simulator.run(); - // wait for tasks to finish - for (Future future : futures) { - future.get(); - } + Assert.assertEquals("errors during call simulation", 0, simulator.getErrors().size()); l4j.info(Arrays.toString(loadBalancer.getHostStats())); @@ -95,10 +70,12 @@ public void run() { @Test public void testEfficiency() throws Exception { // turn down logging (will skew result drastically) - Level logLevel = Logger.getRootLogger().getLevel(); - Logger.getRootLogger().setLevel(Level.WARN); + Logger hostLogger = Logger.getLogger(Host.class); + Level logLevel = hostLogger.getLevel(); + hostLogger.setLevel(Level.WARN); + - SmartConfig smartConfig = new SmartConfig(Arrays.asList("foo", "bar", "baz", "biz")); + SmartConfig smartConfig = new SmartConfig("foo", "bar", "baz", "biz"); LoadBalancer loadBalancer = smartConfig.getLoadBalancer(); @@ -119,9 +96,9 @@ public void testEfficiency() throws Exception { l4j.info(Arrays.toString(loadBalancer.getHostStats())); LogMF.warn(l4j, "per call overhead: {0}µs", perCallOverhead / 1000); - Logger.getRootLogger().setLevel(logLevel); + hostLogger.setLevel(logLevel); - Assert.assertTrue("call overhead too high", perCallOverhead < 150000); // must be less than .15ms + Assert.assertTrue("call overhead too high", perCallOverhead < 500000); // must be less than .5ms } class LBOverheadTask implements Callable { @@ -134,7 +111,7 @@ public LBOverheadTask(LoadBalancer loadBalancer) { @Override public Long call() throws Exception { long start = System.nanoTime(); - Host host = loadBalancer.getTopHost(); + Host host = loadBalancer.getTopHost(null); host.connectionOpened(); host.callComplete(0, false); host.connectionClosed(); diff --git a/src/test/java/com/emc/rest/smart/SmartClientTest.java b/src/test/java/com/emc/rest/smart/SmartClientTest.java index 07f12ee..ea36ee5 100644 --- a/src/test/java/com/emc/rest/smart/SmartClientTest.java +++ b/src/test/java/com/emc/rest/smart/SmartClientTest.java @@ -72,9 +72,9 @@ public void testAtmosOnEcs() throws Exception { String[] endpoints = endpointStr.split(","); final URI serverUri = new URI(endpointStr.split(",")[0]); - List initialHosts = new ArrayList(); + List initialHosts = new ArrayList(); for (String endpoint : endpoints) { - initialHosts.add(new URI(endpoint).getHost()); + initialHosts.add(new Host(new URI(endpoint).getHost())); } SmartConfig smartConfig = new SmartConfig(initialHosts); diff --git a/src/test/java/com/emc/rest/smart/TestHealthCheck.java b/src/test/java/com/emc/rest/smart/TestHealthCheck.java new file mode 100644 index 0000000..e336e6e --- /dev/null +++ b/src/test/java/com/emc/rest/smart/TestHealthCheck.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart; + +import com.emc.util.RequestSimulator; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class TestHealthCheck { + private static final Logger l4j = Logger.getLogger(TestHealthCheck.class); + + @Test + public void testUnhealthyHostIgnored() throws Exception { + String[] hostList = new String[]{"foo", "bar", "baz", "biz"}; + final int callCount = 1000, callDuration = 50; + + SmartConfig smartConfig = new SmartConfig(hostList); + smartConfig.setPollInterval(1); + + final LoadBalancer loadBalancer = smartConfig.getLoadBalancer(); + + // make one host unhealthy + Host foo = loadBalancer.getAllHosts().get(0); + foo.setHealthy(false); + + RequestSimulator simulator = new RequestSimulator(loadBalancer, callCount, callDuration); + simulator.run(); + + Assert.assertEquals("errors during call simulation", 0, simulator.getErrors().size()); + + l4j.info(Arrays.toString(loadBalancer.getHostStats())); + + for (HostStats stats : loadBalancer.getHostStats()) { + if (stats.equals(foo)) { + Assert.assertEquals("unhealthy host should be ignored", 0, stats.getTotalConnections()); + } else { + Assert.assertTrue("unbalanced call count", Math.abs(callCount / (hostList.length - 1) - stats.getTotalConnections()) <= 3); + Assert.assertEquals("average response wrong", callDuration, stats.getResponseQueueAverage()); + } + } + } + + @Test + public void testHealthyUpdate() throws Exception { + String[] hostList = new String[]{"foo", "bar", "baz", "biz"}; + final int callCount = 1000, callDuration = 50; + + SmartConfig smartConfig = new SmartConfig(hostList); + + LoadBalancer loadBalancer = smartConfig.getLoadBalancer(); + + // make one host unhealthy + Host foo = loadBalancer.getAllHosts().get(0); + + TestHostListProvider testProvider = new TestHostListProvider(foo, false); + smartConfig.setHostListProvider(testProvider); + + // first poll should keep list the same and set foo to unhealthy + PollingDaemon poller = new PollingDaemon(smartConfig); + poller.start(); // starting a thread! + Thread.sleep(200); // give poller a chance to run + poller.terminate(); + + Assert.assertFalse(foo.isHealthy()); + Assert.assertEquals(4, loadBalancer.getAllHosts().size()); + + // simulate calls + RequestSimulator simulator = new RequestSimulator(loadBalancer, callCount, callDuration); + simulator.run(); + + Assert.assertEquals("errors during call simulation", 0, simulator.getErrors().size()); + + l4j.info(Arrays.toString(loadBalancer.getHostStats())); + + for (HostStats stats : loadBalancer.getHostStats()) { + if (stats.equals(foo)) { + Assert.assertEquals("unhealthy host should be ignored", 0, stats.getTotalConnections()); + } else { + Assert.assertTrue("unbalanced call count", Math.abs(callCount / (hostList.length - 1) - stats.getTotalConnections()) <= 3); + Assert.assertEquals("average response wrong", callDuration, stats.getResponseQueueAverage()); + } + } + + // second poll should keep list the same and set foo to healthy + testProvider.healthy = true; + poller = new PollingDaemon(smartConfig); + poller.start(); // starting a thread! + Thread.sleep(200); // give poller a chance to run + poller.terminate(); + + Assert.assertTrue(foo.isHealthy()); + Assert.assertEquals(4, loadBalancer.getAllHosts().size()); + + // reset stats and simulate calls + loadBalancer.resetStats(); + simulator = new RequestSimulator(loadBalancer, callCount, callDuration); + simulator.run(); + + Assert.assertEquals("errors during call simulation", 0, simulator.getErrors().size()); + + l4j.info(Arrays.toString(loadBalancer.getHostStats())); + + for (HostStats stats : loadBalancer.getHostStats()) { + Assert.assertTrue("unbalanced call count", Math.abs(callCount / hostList.length - stats.getTotalConnections()) <= 3); + Assert.assertEquals("average response wrong", callDuration, stats.getResponseQueueAverage()); + } + } + + class TestHostListProvider implements HostListProvider { + private Host host; + boolean healthy; + + public TestHostListProvider(Host host, boolean healthy) { + this.host = host; + this.healthy = healthy; + } + + @Override + public List getHostList() { + throw new RuntimeException("no host update"); + } + + @Override + public void runHealthCheck(Host host) { + if (this.host == host && !healthy) throw new RuntimeException("host is unhealthy"); + } + } +} diff --git a/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java new file mode 100644 index 0000000..b795a80 --- /dev/null +++ b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import com.emc.rest.smart.Host; +import com.emc.rest.smart.SmartConfig; +import com.emc.util.TestConfig; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.client.apache4.ApacheHttpClient4; +import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.util.List; +import java.util.Properties; + +public class EcsHostListProviderTest { + public static final String S3_ENDPOINT = "s3.endpoint"; + public static final String S3_ACCESS_KEY = "s3.access_key"; + public static final String S3_SECRET_KEY = "s3.secret_key"; + + public static final String PROXY_URI = "http.proxyUri"; + + @Test + public void testEcsHostListProvider() throws Exception { + Properties properties = TestConfig.getProperties(); + + URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, S3_ENDPOINT)); + String user = TestConfig.getPropertyNotEmpty(properties, S3_ACCESS_KEY); + String secret = TestConfig.getPropertyNotEmpty(properties, S3_SECRET_KEY); + String proxyUri = properties.getProperty(PROXY_URI); + + ClientConfig clientConfig = new DefaultClientConfig(); + if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri); + Client client = ApacheHttpClient4.create(clientConfig); + + SmartConfig smartConfig = new SmartConfig(serverURI.getHost()); + + EcsHostListProvider hostListProvider = new EcsHostListProvider(client, smartConfig.getLoadBalancer(), user, secret); + hostListProvider.setProtocol(serverURI.getScheme()); + hostListProvider.setPort(serverURI.getPort()); + + List hostList = hostListProvider.getHostList(); + + Assert.assertTrue("server list is empty", hostList.size() > 0); + } + + @Test + public void testHealthCheck() throws Exception { + Properties properties = TestConfig.getProperties(); + + URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, S3_ENDPOINT)); + String user = TestConfig.getPropertyNotEmpty(properties, S3_ACCESS_KEY); + String secret = TestConfig.getPropertyNotEmpty(properties, S3_SECRET_KEY); + String proxyUri = properties.getProperty(PROXY_URI); + + ClientConfig clientConfig = new DefaultClientConfig(); + if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri); + Client client = ApacheHttpClient4.create(clientConfig); + + SmartConfig smartConfig = new SmartConfig(serverURI.getHost()); + + EcsHostListProvider hostListProvider = new EcsHostListProvider(client, smartConfig.getLoadBalancer(), user, secret); + hostListProvider.setProtocol(serverURI.getScheme()); + hostListProvider.setPort(serverURI.getPort()); + + for (Host host : hostListProvider.getHostList()) { + hostListProvider.runHealthCheck(host); + } + + try { + hostListProvider.runHealthCheck(new Host("localhost")); + Assert.fail("health check against bad host should fail"); + } catch (Exception e) { + // expected + } + } + + @Test + public void testVdcs() throws Exception { + Properties properties = TestConfig.getProperties(); + + URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, S3_ENDPOINT)); + String user = TestConfig.getPropertyNotEmpty(properties, S3_ACCESS_KEY); + String secret = TestConfig.getPropertyNotEmpty(properties, S3_SECRET_KEY); + String proxyUri = properties.getProperty(PROXY_URI); + + ClientConfig clientConfig = new DefaultClientConfig(); + if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri); + Client client = ApacheHttpClient4.create(clientConfig); + + SmartConfig smartConfig = new SmartConfig(serverURI.getHost()); + + EcsHostListProvider hostListProvider = new EcsHostListProvider(client, smartConfig.getLoadBalancer(), user, secret); + hostListProvider.setProtocol(serverURI.getScheme()); + hostListProvider.setPort(serverURI.getPort()); + + Vdc vdc1 = new Vdc(serverURI.getHost()).withName("vdc1"); + Vdc vdc2 = new Vdc(serverURI.getHost()).withName("vdc2"); + Vdc vdc3 = new Vdc(serverURI.getHost()).withName("vdc3"); + + hostListProvider.withVdcs(vdc1, vdc2, vdc3); + + List hostList = hostListProvider.getHostList(); + + Assert.assertTrue("server list should have at least 3 entries", hostList.size() >= 3); + Assert.assertTrue("VDC1 server list is empty", vdc1.getHosts().size() > 0); + Assert.assertTrue("VDC2 server list is empty", vdc2.getHosts().size() > 0); + Assert.assertTrue("VDC3 server list is empty", vdc3.getHosts().size() > 0); + } +} diff --git a/src/test/java/com/emc/rest/smart/ecs/ListDataNodeTest.java b/src/test/java/com/emc/rest/smart/ecs/ListDataNodeTest.java new file mode 100644 index 0000000..208b82c --- /dev/null +++ b/src/test/java/com/emc/rest/smart/ecs/ListDataNodeTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import org.junit.Assert; +import org.junit.Test; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import java.io.StringReader; +import java.io.StringWriter; + +public class ListDataNodeTest { + @Test + public void testMarshalling() throws Exception { + JAXBContext context = JAXBContext.newInstance(ListDataNode.class); + + String xml = "" + + "" + + "10.241.48.51" + + "10.241.48.52" + + "10.241.48.53" + + "10.241.48.54" + + "1.2.0.0.60152.d32b519" + + ""; + + ListDataNode listDataNode = new ListDataNode(); + listDataNode.getDataNodes().add("10.241.48.51"); + listDataNode.getDataNodes().add("10.241.48.52"); + listDataNode.getDataNodes().add("10.241.48.53"); + listDataNode.getDataNodes().add("10.241.48.54"); + listDataNode.setVersionInfo("1.2.0.0.60152.d32b519"); + + // unmarshall and compare to object + Unmarshaller unmarshaller = context.createUnmarshaller(); + ListDataNode unmarshalledObject = (ListDataNode) unmarshaller.unmarshal(new StringReader(xml)); + + Assert.assertEquals(listDataNode.getDataNodes(), unmarshalledObject.getDataNodes()); + Assert.assertEquals(listDataNode.getVersionInfo(), unmarshalledObject.getVersionInfo()); + + // marshall and compare XML + Marshaller marshaller = context.createMarshaller(); + StringWriter writer = new StringWriter(); + marshaller.marshal(listDataNode, writer); + + Assert.assertEquals(xml, writer.toString()); + } +} diff --git a/src/test/java/com/emc/util/RequestSimulator.java b/src/test/java/com/emc/util/RequestSimulator.java new file mode 100644 index 0000000..7092f5f --- /dev/null +++ b/src/test/java/com/emc/util/RequestSimulator.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.util; + +import com.emc.rest.smart.Host; +import com.emc.rest.smart.LoadBalancer; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class RequestSimulator implements Runnable { + private LoadBalancer loadBalancer; + private int callCount; + private int callDuration; + private RequestExecutor requestExecutor; + private List errors = new ArrayList(); + + public RequestSimulator(LoadBalancer loadBalancer, int callCount, int callDuration) { + this.loadBalancer = loadBalancer; + this.callCount = callCount; + this.callDuration = callDuration; + } + + @Override + public void run() { + final Random random = new Random(); + + // simulate callCount successful calls with identical response times + ExecutorService executorService = Executors.newFixedThreadPool(10); + List futures = new ArrayList(); + for (int i = 0; i < callCount; i++) { + futures.add(executorService.submit(new Runnable() { + @Override + public void run() { + int waitMs; + synchronized (random) { + waitMs = random.nextInt(20); + } + try { + Thread.sleep(waitMs); + } catch (InterruptedException e) { + e.printStackTrace(); + } + Host host = loadBalancer.getTopHost(null); + host.connectionOpened(); + try { + if (requestExecutor != null) requestExecutor.execute(host); + host.callComplete(callDuration, false); + } catch (Throwable t) { + host.callComplete(callDuration, true); + } + host.connectionClosed(); + } + })); + } + + // wait for tasks to finish + for (Future future : futures) { + try { + future.get(); + } catch (Throwable t) { + errors.add(t); + } + } + } + + public RequestSimulator withRequestExecutor(RequestExecutor requestExecutor) { + this.requestExecutor = requestExecutor; + return this; + } + + public List getErrors() { + return errors; + } + + public interface RequestExecutor { + void execute(Host host); + } +}