Prevent bad reads from Edda marking ELBs as failing cross-zone balancing

Netflix · Dec 13, 2016 · 04c73cc · 04c73cc
1 parent 107bf20
commit 04c73cc
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 7 deletions.
diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/CrossZoneLoadBalancing.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/CrossZoneLoadBalancing.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.netflix.simianarmy.client.MonkeyRestClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -70,12 +71,17 @@ public CrossZoneLoadBalancing(AWSCredentialsProvider awsCredentialsProvider) {
     public Conformity check(Cluster cluster) {
         Collection<String> failedComponents = Lists.newArrayList();
         for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) {
-            for (String lbName : getLoadBalancerNamesForAsg(cluster.getRegion(), asg.getName())) {
-                if (!isCrossZoneLoadBalancingEnabled(cluster.getRegion(), lbName)) {
-                    LOGGER.info(String.format("ELB %s in %s does not have cross-zone load balancing enabled",
-                            lbName, cluster.getRegion()));
-                    failedComponents.add(lbName);
+            try {
+                for (String lbName : getLoadBalancerNamesForAsg(cluster.getRegion(), asg.getName())) {
+                    if (!isCrossZoneLoadBalancingEnabled(cluster.getRegion(), lbName)) {
+                        LOGGER.info(String.format("ELB %s in %s does not have cross-zone load balancing enabled",
+                                    lbName, cluster.getRegion()));
+                        failedComponents.add(lbName);
+                    }
                 }
+            } catch (MonkeyRestClient.DataReadException e) {
+                LOGGER.error(String.format("Transient error reading ELB for %s in %s - skipping this check",
+                             asg.getName(), cluster.getRegion()), e);
             }
         }
         return new Conformity(getName(), failedComponents);

diff --git a/src/main/java/com/netflix/simianarmy/client/MonkeyRestClient.java b/src/main/java/com/netflix/simianarmy/client/MonkeyRestClient.java
@@ -51,7 +51,7 @@ public MonkeyRestClient(int timeout, int maxRetries, int retryInterval) {
         Validate.isTrue(timeout >= 0);
         Validate.isTrue(maxRetries >= 0);
         Validate.isTrue(retryInterval > 0);
-        
+
         RequestConfig config = RequestConfig.custom()
             .setConnectTimeout(timeout)
             .build();
@@ -87,7 +87,7 @@ public JsonNode getJsonNodeFromUrl(String url) throws IOException {
         if (code == 404) {
             return null;
         } else if (code >= 300 || code < 200) {
-            throw new RuntimeException(String.format("Response code %d from url %s: %s", code, url, jsonContent));
+            throw new DataReadException(code, url, jsonContent);
         }
 
         JsonNode result;
@@ -107,4 +107,10 @@ public JsonNode getJsonNodeFromUrl(String url) throws IOException {
      * @return the base url in the region
      */
     public abstract String getBaseUrl(String region);
+
+    public static class DataReadException extends RuntimeException {
+        public DataReadException(int code, String url, String jsonContent) {
+            super(String.format("Response code %d from url %s: %s", code, url, jsonContent));
+        }
+    }
 }