diff --git a/src/main/java/com/emc/rest/smart/Host.java b/src/main/java/com/emc/rest/smart/Host.java index 723f09c..4cd3859 100644 --- a/src/main/java/com/emc/rest/smart/Host.java +++ b/src/main/java/com/emc/rest/smart/Host.java @@ -34,16 +34,19 @@ /** * Some basic statements about response index calculation: *

- * - lower response index means the host is more likely to be used - * - should be based primarily on number of open connections to the host - * - an error will mark the host as unhealthy for errorWaitTime milliseconds - * - multiple consecutive errors compound the unhealthy (cool down) period up to 8x the errorWaitTime + *

*/ public class Host implements HostStats { private static final Logger l4j = Logger.getLogger(Host.class); public static final int DEFAULT_ERROR_WAIT_MS = 1500; public static final int LOG_DELAY = 60000; // 1 minute + public static final int MAX_COOL_DOWN_EXP = 4; private String name; private boolean healthy = true; @@ -74,8 +77,11 @@ public synchronized void connectionClosed() { // Just in case our stats get out of whack somehow, make sure people know about it if (openConnections < 0) { - if (System.currentTimeMillis() - lastLogTime > LOG_DELAY) + long currentTime = System.currentTimeMillis(); + if (currentTime - lastLogTime > LOG_DELAY) { LogMF.warn(l4j, "openConnections for host %s is %d !", this, openConnections); + lastLogTime = currentTime; + } } } @@ -98,9 +104,9 @@ public boolean isHealthy() { if (!healthy) return false; else if (consecutiveErrors == 0) return true; else { - long coolDownPower = consecutiveErrors > 3 ? 3 : consecutiveErrors - 1; + long coolDownExp = consecutiveErrors > MAX_COOL_DOWN_EXP ? MAX_COOL_DOWN_EXP : consecutiveErrors - 1; long msSinceLastUse = System.currentTimeMillis() - lastConnectionTime; - long errorCoolDown = (long) Math.pow(2, coolDownPower) * errorWaitTime; + long errorCoolDown = (long) Math.pow(2, coolDownExp) * errorWaitTime; return msSinceLastUse > errorCoolDown; } } diff --git a/src/main/java/com/emc/rest/smart/SmartClientFactory.java b/src/main/java/com/emc/rest/smart/SmartClientFactory.java index 44eaffe..f3d8d33 100644 --- a/src/main/java/com/emc/rest/smart/SmartClientFactory.java +++ b/src/main/java/com/emc/rest/smart/SmartClientFactory.java @@ -36,9 +36,6 @@ import com.sun.jersey.core.impl.provider.entity.ByteArrayProvider; import com.sun.jersey.core.impl.provider.entity.FileProvider; import com.sun.jersey.core.impl.provider.entity.InputStreamProvider; -import org.apache.http.impl.client.AbstractHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.PoolingClientConnectionManager; import org.apache.log4j.Logger; public final class SmartClientFactory { @@ -137,7 +134,7 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi ClientConfig clientConfig = new DefaultClientConfig(); // set up multi-threaded connection pool - PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); + org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // 200 maximum active connections (should be more than enough for any JVM instance) connectionManager.setDefaultMaxPerRoute(200); connectionManager.setMaxTotal(200); @@ -151,11 +148,18 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi if (smartConfig.getProxyPass() != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_PASSWORD, smartConfig.getProxyPass()); + // pass in jersey parameters from calling code (allows customization of client) + for (String propName : smartConfig.getProperties().keySet()) { + clientConfig.getProperties().put(propName, smartConfig.getProperty(propName)); + } + ApacheHttpClient4Handler handler = ApacheHttpClient4.create(clientConfig).getClientHandler(); // disable the retry handler if necessary - if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null) - ((AbstractHttpClient) handler.getHttpClient()).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); + if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null) { + org.apache.http.impl.client.AbstractHttpClient httpClient = (org.apache.http.impl.client.AbstractHttpClient) handler.getHttpClient(); + httpClient.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false)); + } return handler; } diff --git a/src/main/java/com/emc/rest/smart/SmartFilter.java b/src/main/java/com/emc/rest/smart/SmartFilter.java index 7e72f14..1bfa5b5 100644 --- a/src/main/java/com/emc/rest/smart/SmartFilter.java +++ b/src/main/java/com/emc/rest/smart/SmartFilter.java @@ -30,8 +30,6 @@ import com.sun.jersey.api.client.ClientRequest; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.filter.ClientFilter; -import org.apache.http.HttpHost; -import org.apache.http.client.utils.URIUtils; import java.io.FilterInputStream; import java.io.IOException; @@ -62,7 +60,8 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio // replace the host in the request URI uri = request.getURI(); try { - uri = URIUtils.rewriteURI(uri, new HttpHost(host.getName(), uri.getPort(), uri.getScheme())); + org.apache.http.HttpHost httpHost = new org.apache.http.HttpHost(host.getName(), uri.getPort(), uri.getScheme()); + uri = org.apache.http.client.utils.URIUtils.rewriteURI(uri, httpHost); } catch (URISyntaxException e) { throw new RuntimeException("load-balanced host generated invalid URI", e); } diff --git a/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java index 8ac229d..a8e8533 100644 --- a/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java +++ b/src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java @@ -99,7 +99,18 @@ public List getHostList() { @Override public void runHealthCheck(Host host) { // header is workaround for STORAGE-1833 - client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x").get(String.class); + PingResponse response = client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x") + .get(PingResponse.class); + + if (host instanceof VdcHost) { + PingItem.Status status = PingItem.Status.OFF; + if (response != null && response.getPingItemMap() != null) { + PingItem pingItem = response.getPingItemMap().get(PingItem.MAINTENANCE_MODE); + if (pingItem != null) status = pingItem.getStatus(); + } + if (status == PingItem.Status.ON) ((VdcHost) host).setMaintenanceMode(true); + else ((VdcHost) host).setMaintenanceMode(false); + } } @Override diff --git a/src/main/java/com/emc/rest/smart/ecs/PingItem.java b/src/main/java/com/emc/rest/smart/ecs/PingItem.java new file mode 100644 index 0000000..75034d7 --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/PingItem.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlEnum; + +public class PingItem { + public static final String MAINTENANCE_MODE = "MAINTENANCE_MODE"; + + String name; + Status status; + String text; + + @XmlElement(name = "Name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @XmlElement(name = "Status") + public Status getStatus() { + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + + @XmlElement(name = "Text") + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + @XmlEnum + public static enum Status { + OFF, UNKNOWN, ON + } +} diff --git a/src/main/java/com/emc/rest/smart/ecs/PingResponse.java b/src/main/java/com/emc/rest/smart/ecs/PingResponse.java new file mode 100644 index 0000000..9bc72cc --- /dev/null +++ b/src/main/java/com/emc/rest/smart/ecs/PingResponse.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.rest.smart.ecs; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@XmlRootElement(name = "PingList") +public class PingResponse { + Map pingItemMap; + + @XmlElement(name = "PingItem") + public List getPingItems() { + if (pingItemMap == null) return null; + return new ArrayList(pingItemMap.values()); + } + + public void setPingItems(List pingItems) { + if (pingItems != null) { + pingItemMap = new HashMap(); + for (PingItem item : pingItems) { + pingItemMap.put(item.getName(), item); + } + } + } + + @XmlTransient + public Map getPingItemMap() { + return pingItemMap; + } + + public void setPingItemMap(Map pingItemMap) { + this.pingItemMap = pingItemMap; + } +} diff --git a/src/main/java/com/emc/rest/smart/ecs/VdcHost.java b/src/main/java/com/emc/rest/smart/ecs/VdcHost.java index d51274c..a819c16 100644 --- a/src/main/java/com/emc/rest/smart/ecs/VdcHost.java +++ b/src/main/java/com/emc/rest/smart/ecs/VdcHost.java @@ -30,12 +30,18 @@ public class VdcHost extends Host { private Vdc vdc; + private boolean maintenanceMode; public VdcHost(Vdc vdc, String name) { super(name); this.vdc = vdc; } + @Override + public boolean isHealthy() { + return !isMaintenanceMode() && super.isHealthy(); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -63,4 +69,12 @@ public String toString() { public Vdc getVdc() { return vdc; } + + public boolean isMaintenanceMode() { + return maintenanceMode; + } + + public void setMaintenanceMode(boolean maintenanceMode) { + this.maintenanceMode = maintenanceMode; + } } diff --git a/src/test/java/com/emc/rest/smart/HostTest.java b/src/test/java/com/emc/rest/smart/HostTest.java index 5565ff8..fc30426 100644 --- a/src/test/java/com/emc/rest/smart/HostTest.java +++ b/src/test/java/com/emc/rest/smart/HostTest.java @@ -121,4 +121,27 @@ public void testHost() throws Exception { Assert.assertEquals(0, host.getResponseIndex()); Assert.assertTrue(host.isHealthy()); } + + @Test + public void testErrorWaitLimit() throws Exception { + Host host = new Host("bar"); + host.setErrorWaitTime(100); // don't want this test to take forever + + Assert.assertTrue(host.isHealthy()); + + // 8 consecutive errors + long errors = 8; + for (int i = 0; i < errors; i++) { + host.connectionOpened(); + host.callComplete(true); + host.connectionClosed(); + } + + Assert.assertEquals(errors, host.getConsecutiveErrors()); + long maxCoolDownMs = host.getErrorWaitTime() * (long) Math.pow(2, Host.MAX_COOL_DOWN_EXP) + 10; // add a few ms + + Thread.sleep(maxCoolDownMs); + + Assert.assertTrue(host.isHealthy()); + } } diff --git a/src/test/java/com/emc/rest/smart/SmartClientTest.java b/src/test/java/com/emc/rest/smart/SmartClientTest.java index ea36ee5..0afaafc 100644 --- a/src/test/java/com/emc/rest/smart/SmartClientTest.java +++ b/src/test/java/com/emc/rest/smart/SmartClientTest.java @@ -28,9 +28,14 @@ import com.emc.util.TestConfig; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; import org.apache.commons.codec.binary.Base64; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.HttpConnectionParams; +import org.apache.http.params.HttpParams; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Assume; @@ -42,9 +47,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; public class SmartClientTest { @@ -105,6 +108,36 @@ public void run() { Assert.assertEquals("at least one task failed", 100, successCount.intValue()); } + @Test + public void testConnTimeout() throws Exception { + int CONNECTION_TIMEOUT_MILLIS = 10000; // 10 seconds + + HttpParams httpParams = new BasicHttpParams(); + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS); + + SmartConfig smartConfig = new SmartConfig("10.4.4.180"); + smartConfig.setProperty(ApacheHttpClient4Config.PROPERTY_HTTP_PARAMS, httpParams); + + final Client client = SmartClientFactory.createStandardClient(smartConfig); + + Future future = Executors.newSingleThreadExecutor().submit(new Runnable() { + @Override + public void run() { + client.resource("http://10.4.4.180:9020/?ping").get(String.class); + Assert.fail("response was not expected; choose an IP that is not in use"); + } + }); + + try { + future.get(CONNECTION_TIMEOUT_MILLIS + 1000, TimeUnit.MILLISECONDS); // give an extra second leeway + } catch (TimeoutException e) { + Assert.fail("connection did not timeout"); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof ClientHandlerException); + Assert.assertTrue(e.getMessage().contains("timed out")); + } + } + private void getServiceInfo(Client client, URI serverUri, String uid, String secretKey) { String path = "/rest/service"; String date = getDateFormat().format(new Date()); diff --git a/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java index b795a80..6141be6 100644 --- a/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java +++ b/src/test/java/com/emc/rest/smart/ecs/EcsHostListProviderTest.java @@ -95,6 +95,18 @@ public void testHealthCheck() throws Exception { hostListProvider.runHealthCheck(host); } + // test non-VDC host + Host host = new Host(serverURI.getHost()); + hostListProvider.runHealthCheck(host); + Assert.assertTrue(host.isHealthy()); + + // test VDC host + Vdc vdc = new Vdc(serverURI.getHost()); + VdcHost vdcHost = vdc.getHosts().get(0); + hostListProvider.runHealthCheck(vdcHost); + Assert.assertTrue(vdcHost.isHealthy()); + Assert.assertFalse(vdcHost.isMaintenanceMode()); + try { hostListProvider.runHealthCheck(new Host("localhost")); Assert.fail("health check against bad host should fail"); @@ -103,6 +115,42 @@ public void testHealthCheck() throws Exception { } } + @Test + public void testMaintenanceMode() { + Vdc vdc = new Vdc("foo.com"); + VdcHost host = vdc.getHosts().get(0); + + // assert the host is healthy first + Assert.assertTrue(host.isHealthy()); + + // maintenance mode should make the host appear offline + host.setMaintenanceMode(true); + Assert.assertFalse(host.isHealthy()); + + host.setMaintenanceMode(false); + Assert.assertTrue(host.isHealthy()); + } + + @Test + public void testPing() throws Exception { + Properties properties = TestConfig.getProperties(); + + URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, S3_ENDPOINT)); + String proxyUri = properties.getProperty(PROXY_URI); + + ClientConfig clientConfig = new DefaultClientConfig(); + if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri); + Client client = ApacheHttpClient4.create(clientConfig); + + String portStr = serverURI.getPort() > 0 ? ":" + serverURI.getPort() : ""; + + PingResponse response = client.resource( + String.format("%s://%s%s/?ping", serverURI.getScheme(), serverURI.getHost(), portStr)) + .header("x-emc-namespace", "foo").get(PingResponse.class); + Assert.assertNotNull(response); + Assert.assertEquals(PingItem.Status.OFF, response.getPingItemMap().get(PingItem.MAINTENANCE_MODE).getStatus()); + } + @Test public void testVdcs() throws Exception { Properties properties = TestConfig.getProperties(); diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index 49e1fc8..165cc61 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -36,6 +36,9 @@ + + +