diff --git a/src/main/java/com/emc/rest/smart/Host.java b/src/main/java/com/emc/rest/smart/Host.java
index 723f09c..4cd3859 100644
--- a/src/main/java/com/emc/rest/smart/Host.java
+++ b/src/main/java/com/emc/rest/smart/Host.java
@@ -34,16 +34,19 @@
/**
* Some basic statements about response index calculation:
*
- * - lower response index means the host is more likely to be used
- * - should be based primarily on number of open connections to the host
- * - an error will mark the host as unhealthy for errorWaitTime
milliseconds
- * - multiple consecutive errors compound the unhealthy (cool down) period up to 8x the errorWaitTime
+ *
+ * - lower response index means the host is more likely to be used
+ * - should be based primarily on number of open connections to the host
+ * - an error will mark the host as unhealthy for
errorWaitTime
milliseconds
+ * - multiple consecutive errors compound the unhealthy (cool down) period up to 16x the errorWaitTime
+ *
*/
public class Host implements HostStats {
private static final Logger l4j = Logger.getLogger(Host.class);
public static final int DEFAULT_ERROR_WAIT_MS = 1500;
public static final int LOG_DELAY = 60000; // 1 minute
+ public static final int MAX_COOL_DOWN_EXP = 4;
private String name;
private boolean healthy = true;
@@ -74,8 +77,11 @@ public synchronized void connectionClosed() {
// Just in case our stats get out of whack somehow, make sure people know about it
if (openConnections < 0) {
- if (System.currentTimeMillis() - lastLogTime > LOG_DELAY)
+ long currentTime = System.currentTimeMillis();
+ if (currentTime - lastLogTime > LOG_DELAY) {
LogMF.warn(l4j, "openConnections for host %s is %d !", this, openConnections);
+ lastLogTime = currentTime;
+ }
}
}
@@ -98,9 +104,9 @@ public boolean isHealthy() {
if (!healthy) return false;
else if (consecutiveErrors == 0) return true;
else {
- long coolDownPower = consecutiveErrors > 3 ? 3 : consecutiveErrors - 1;
+ long coolDownExp = consecutiveErrors > MAX_COOL_DOWN_EXP ? MAX_COOL_DOWN_EXP : consecutiveErrors - 1;
long msSinceLastUse = System.currentTimeMillis() - lastConnectionTime;
- long errorCoolDown = (long) Math.pow(2, coolDownPower) * errorWaitTime;
+ long errorCoolDown = (long) Math.pow(2, coolDownExp) * errorWaitTime;
return msSinceLastUse > errorCoolDown;
}
}
diff --git a/src/main/java/com/emc/rest/smart/SmartClientFactory.java b/src/main/java/com/emc/rest/smart/SmartClientFactory.java
index 44eaffe..f3d8d33 100644
--- a/src/main/java/com/emc/rest/smart/SmartClientFactory.java
+++ b/src/main/java/com/emc/rest/smart/SmartClientFactory.java
@@ -36,9 +36,6 @@
import com.sun.jersey.core.impl.provider.entity.ByteArrayProvider;
import com.sun.jersey.core.impl.provider.entity.FileProvider;
import com.sun.jersey.core.impl.provider.entity.InputStreamProvider;
-import org.apache.http.impl.client.AbstractHttpClient;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
-import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.log4j.Logger;
public final class SmartClientFactory {
@@ -137,7 +134,7 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi
ClientConfig clientConfig = new DefaultClientConfig();
// set up multi-threaded connection pool
- PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
+ org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager = new org.apache.http.impl.conn.PoolingClientConnectionManager();
// 200 maximum active connections (should be more than enough for any JVM instance)
connectionManager.setDefaultMaxPerRoute(200);
connectionManager.setMaxTotal(200);
@@ -151,11 +148,18 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi
if (smartConfig.getProxyPass() != null)
clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_PASSWORD, smartConfig.getProxyPass());
+ // pass in jersey parameters from calling code (allows customization of client)
+ for (String propName : smartConfig.getProperties().keySet()) {
+ clientConfig.getProperties().put(propName, smartConfig.getProperty(propName));
+ }
+
ApacheHttpClient4Handler handler = ApacheHttpClient4.create(clientConfig).getClientHandler();
// disable the retry handler if necessary
- if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null)
- ((AbstractHttpClient) handler.getHttpClient()).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
+ if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null) {
+ org.apache.http.impl.client.AbstractHttpClient httpClient = (org.apache.http.impl.client.AbstractHttpClient) handler.getHttpClient();
+ httpClient.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false));
+ }
return handler;
}
diff --git a/src/main/java/com/emc/rest/smart/SmartFilter.java b/src/main/java/com/emc/rest/smart/SmartFilter.java
index 7e72f14..1bfa5b5 100644
--- a/src/main/java/com/emc/rest/smart/SmartFilter.java
+++ b/src/main/java/com/emc/rest/smart/SmartFilter.java
@@ -30,8 +30,6 @@
import com.sun.jersey.api.client.ClientRequest;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.filter.ClientFilter;
-import org.apache.http.HttpHost;
-import org.apache.http.client.utils.URIUtils;
import java.io.FilterInputStream;
import java.io.IOException;
@@ -62,7 +60,8 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio
// replace the host in the request
URI uri = request.getURI();
try {
- uri = URIUtils.rewriteURI(uri, new HttpHost(host.getName(), uri.getPort(), uri.getScheme()));
+ org.apache.http.HttpHost httpHost = new org.apache.http.HttpHost(host.getName(), uri.getPort(), uri.getScheme());
+ uri = org.apache.http.client.utils.URIUtils.rewriteURI(uri, httpHost);
} catch (URISyntaxException e) {
throw new RuntimeException("load-balanced host generated invalid URI", e);
}
diff --git a/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java
index 8ac229d..a8e8533 100644
--- a/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java
+++ b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java
@@ -99,7 +99,18 @@ public List getHostList() {
@Override
public void runHealthCheck(Host host) {
// header is workaround for STORAGE-1833
- client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x").get(String.class);
+ PingResponse response = client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x")
+ .get(PingResponse.class);
+
+ if (host instanceof VdcHost) {
+ PingItem.Status status = PingItem.Status.OFF;
+ if (response != null && response.getPingItemMap() != null) {
+ PingItem pingItem = response.getPingItemMap().get(PingItem.MAINTENANCE_MODE);
+ if (pingItem != null) status = pingItem.getStatus();
+ }
+ if (status == PingItem.Status.ON) ((VdcHost) host).setMaintenanceMode(true);
+ else ((VdcHost) host).setMaintenanceMode(false);
+ }
}
@Override
diff --git a/src/main/java/com/emc/rest/smart/ecs/PingItem.java b/src/main/java/com/emc/rest/smart/ecs/PingItem.java
new file mode 100644
index 0000000..75034d7
--- /dev/null
+++ b/src/main/java/com/emc/rest/smart/ecs/PingItem.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2015, EMC Corporation.
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * + Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * + Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * + The name of EMC Corporation may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.emc.rest.smart.ecs;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
+
+public class PingItem {
+ public static final String MAINTENANCE_MODE = "MAINTENANCE_MODE";
+
+ String name;
+ Status status;
+ String text;
+
+ @XmlElement(name = "Name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @XmlElement(name = "Status")
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ @XmlElement(name = "Text")
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ @XmlEnum
+ public static enum Status {
+ OFF, UNKNOWN, ON
+ }
+}
diff --git a/src/main/java/com/emc/rest/smart/ecs/PingResponse.java b/src/main/java/com/emc/rest/smart/ecs/PingResponse.java
new file mode 100644
index 0000000..9bc72cc
--- /dev/null
+++ b/src/main/java/com/emc/rest/smart/ecs/PingResponse.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, EMC Corporation.
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * + Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * + Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * + The name of EMC Corporation may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.emc.rest.smart.ecs;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@XmlRootElement(name = "PingList")
+public class PingResponse {
+ Map pingItemMap;
+
+ @XmlElement(name = "PingItem")
+ public List getPingItems() {
+ if (pingItemMap == null) return null;
+ return new ArrayList(pingItemMap.values());
+ }
+
+ public void setPingItems(List pingItems) {
+ if (pingItems != null) {
+ pingItemMap = new HashMap();
+ for (PingItem item : pingItems) {
+ pingItemMap.put(item.getName(), item);
+ }
+ }
+ }
+
+ @XmlTransient
+ public Map getPingItemMap() {
+ return pingItemMap;
+ }
+
+ public void setPingItemMap(Map pingItemMap) {
+ this.pingItemMap = pingItemMap;
+ }
+}
diff --git a/src/main/java/com/emc/rest/smart/ecs/VdcHost.java b/src/main/java/com/emc/rest/smart/ecs/VdcHost.java
index d51274c..a819c16 100644
--- a/src/main/java/com/emc/rest/smart/ecs/VdcHost.java
+++ b/src/main/java/com/emc/rest/smart/ecs/VdcHost.java
@@ -30,12 +30,18 @@
public class VdcHost extends Host {
private Vdc vdc;
+ private boolean maintenanceMode;
public VdcHost(Vdc vdc, String name) {
super(name);
this.vdc = vdc;
}
+ @Override
+ public boolean isHealthy() {
+ return !isMaintenanceMode() && super.isHealthy();
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -63,4 +69,12 @@ public String toString() {
public Vdc getVdc() {
return vdc;
}
+
+ public boolean isMaintenanceMode() {
+ return maintenanceMode;
+ }
+
+ public void setMaintenanceMode(boolean maintenanceMode) {
+ this.maintenanceMode = maintenanceMode;
+ }
}
diff --git a/src/test/java/com/emc/rest/smart/HostTest.java b/src/test/java/com/emc/rest/smart/HostTest.java
index 5565ff8..fc30426 100644
--- a/src/test/java/com/emc/rest/smart/HostTest.java
+++ b/src/test/java/com/emc/rest/smart/HostTest.java
@@ -121,4 +121,27 @@ public void testHost() throws Exception {
Assert.assertEquals(0, host.getResponseIndex());
Assert.assertTrue(host.isHealthy());
}
+
+ @Test
+ public void testErrorWaitLimit() throws Exception {
+ Host host = new Host("bar");
+ host.setErrorWaitTime(100); // don't want this test to take forever
+
+ Assert.assertTrue(host.isHealthy());
+
+ // 8 consecutive errors
+ long errors = 8;
+ for (int i = 0; i < errors; i++) {
+ host.connectionOpened();
+ host.callComplete(true);
+ host.connectionClosed();
+ }
+
+ Assert.assertEquals(errors, host.getConsecutiveErrors());
+ long maxCoolDownMs = host.getErrorWaitTime() * (long) Math.pow(2, Host.MAX_COOL_DOWN_EXP) + 10; // add a few ms
+
+ Thread.sleep(maxCoolDownMs);
+
+ Assert.assertTrue(host.isHealthy());
+ }
}
diff --git a/src/test/java/com/emc/rest/smart/SmartClientTest.java b/src/test/java/com/emc/rest/smart/SmartClientTest.java
index ea36ee5..0afaafc 100644
--- a/src/test/java/com/emc/rest/smart/SmartClientTest.java
+++ b/src/test/java/com/emc/rest/smart/SmartClientTest.java
@@ -28,9 +28,14 @@
import com.emc.util.TestConfig;
import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config;
import org.apache.commons.codec.binary.Base64;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Assume;
@@ -42,9 +47,7 @@
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
public class SmartClientTest {
@@ -105,6 +108,36 @@ public void run() {
Assert.assertEquals("at least one task failed", 100, successCount.intValue());
}
+ @Test
+ public void testConnTimeout() throws Exception {
+ int CONNECTION_TIMEOUT_MILLIS = 10000; // 10 seconds
+
+ HttpParams httpParams = new BasicHttpParams();
+ HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS);
+
+ SmartConfig smartConfig = new SmartConfig("10.4.4.180");
+ smartConfig.setProperty(ApacheHttpClient4Config.PROPERTY_HTTP_PARAMS, httpParams);
+
+ final Client client = SmartClientFactory.createStandardClient(smartConfig);
+
+ Future future = Executors.newSingleThreadExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ client.resource("http://10.4.4.180:9020/?ping").get(String.class);
+ Assert.fail("response was not expected; choose an IP that is not in use");
+ }
+ });
+
+ try {
+ future.get(CONNECTION_TIMEOUT_MILLIS + 1000, TimeUnit.MILLISECONDS); // give an extra second leeway
+ } catch (TimeoutException e) {
+ Assert.fail("connection did not timeout");
+ } catch (ExecutionException e) {
+ Assert.assertTrue(e.getCause() instanceof ClientHandlerException);
+ Assert.assertTrue(e.getMessage().contains("timed out"));
+ }
+ }
+
private void getServiceInfo(Client client, URI serverUri, String uid, String secretKey) {
String path = "/rest/service";
String date = getDateFormat().format(new Date());
diff --git a/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java
index b795a80..6141be6 100644
--- a/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java
+++ b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java
@@ -95,6 +95,18 @@ public void testHealthCheck() throws Exception {
hostListProvider.runHealthCheck(host);
}
+ // test non-VDC host
+ Host host = new Host(serverURI.getHost());
+ hostListProvider.runHealthCheck(host);
+ Assert.assertTrue(host.isHealthy());
+
+ // test VDC host
+ Vdc vdc = new Vdc(serverURI.getHost());
+ VdcHost vdcHost = vdc.getHosts().get(0);
+ hostListProvider.runHealthCheck(vdcHost);
+ Assert.assertTrue(vdcHost.isHealthy());
+ Assert.assertFalse(vdcHost.isMaintenanceMode());
+
try {
hostListProvider.runHealthCheck(new Host("localhost"));
Assert.fail("health check against bad host should fail");
@@ -103,6 +115,42 @@ public void testHealthCheck() throws Exception {
}
}
+ @Test
+ public void testMaintenanceMode() {
+ Vdc vdc = new Vdc("foo.com");
+ VdcHost host = vdc.getHosts().get(0);
+
+ // assert the host is healthy first
+ Assert.assertTrue(host.isHealthy());
+
+ // maintenance mode should make the host appear offline
+ host.setMaintenanceMode(true);
+ Assert.assertFalse(host.isHealthy());
+
+ host.setMaintenanceMode(false);
+ Assert.assertTrue(host.isHealthy());
+ }
+
+ @Test
+ public void testPing() throws Exception {
+ Properties properties = TestConfig.getProperties();
+
+ URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, S3_ENDPOINT));
+ String proxyUri = properties.getProperty(PROXY_URI);
+
+ ClientConfig clientConfig = new DefaultClientConfig();
+ if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri);
+ Client client = ApacheHttpClient4.create(clientConfig);
+
+ String portStr = serverURI.getPort() > 0 ? ":" + serverURI.getPort() : "";
+
+ PingResponse response = client.resource(
+ String.format("%s://%s%s/?ping", serverURI.getScheme(), serverURI.getHost(), portStr))
+ .header("x-emc-namespace", "foo").get(PingResponse.class);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(PingItem.Status.OFF, response.getPingItemMap().get(PingItem.MAINTENANCE_MODE).getStatus());
+ }
+
@Test
public void testVdcs() throws Exception {
Properties properties = TestConfig.getProperties();
diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml
index 49e1fc8..165cc61 100644
--- a/src/test/resources/log4j.xml
+++ b/src/test/resources/log4j.xml
@@ -36,6 +36,9 @@
+
+
+