fix: skip unavailable clouds

plugin/src/main/java/jenkins/plugins/openstack/compute/JCloudsCloud.java

@@ -254,14 +254,23 @@ private void injectReferenceIntoTemplates() {
             return queue; // more slaves then declared - no need to query openstack
         }
 
-
-        final List<Server> runningNodes = getOpenstack().getRunningNodes();
-
-        int serverCount = runningNodes.size();
-        if (serverCount >= globalMax) {
-            return queue; // more servers than needed - no need to proceed any further
-        }
-
+         // Check the number of current servers
+         int serverCount = 0;
+         List<Server> runningNodes = new ArrayList<Server>();
+         try {
+             // get the running nodes
+             runningNodes = getOpenstack().getRunningNodes();
+
+             serverCount = runningNodes.size();
+             if (serverCount >= globalMax) {
+                 return queue; // more servers than needed - no need to proceed any further
+             }
+           }  catch (JCloudsCloud.LoginFailure ex) {
+            LOGGER.log(Level.WARNING, "Login failure: " + ex.getMessage());
+            return queue;
+        } 
+
+
         int globalCapacity = globalMax - Math.max(nodeCount, serverCount);
         assert globalCapacity > 0;
 
@@ -299,8 +308,19 @@ public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excess
 
             final JCloudsSlaveTemplate template = templateProvider.poll();
             if (template == null) {
-                LOGGER.info("Instance cap exceeded for cloud " + name + " while provisioning for label " + label);
-                break;
+                // two cases 
+                // cloud authentication issue or Instance cap exceeded 
+                try {
+                    // try to authenticate 
+                    getOpenstack();
+                    // if okay, then the problem is related to the instance cap 
+                    LOGGER.info("Instance cap exceeded for cloud " + name + " while provisioning for label " + label);
+                    break;
+                  } catch (JCloudsCloud.LoginFailure ex) {
+                   // no need to log here because it has already been logged in the getAvailableTemplates method. 
+                   break; 
+               } 
+
             }
 
             LOGGER.fine("Provisioning slave for " + label + " from template " + template.getName());
@@ -313,9 +333,13 @@ public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excess
 
             excessWorkload -= numExecutors;
         }
+
+
         return plannedNodeList;
     }
 
+
+
     private static final class NodeCallable implements Callable<Node> {
         private final JCloudsCloud cloud;
         private final JCloudsSlaveTemplate template;
@@ -614,7 +638,8 @@ public ProvisioningFailedException(String msg) {
         }
     }
 
-    /*package*/ static final class LoginFailure extends RuntimeException {
+    /*package*/ 
+    public static final class LoginFailure extends RuntimeException {
 
         private static final long serialVersionUID = 4085466675398031930L;
 
@@ -626,7 +651,7 @@ private LoginFailure(String name, AuthenticationException ex) {
             super("Failure to authenticate for cloud " + name + ": " + ex.toString());
         }
 
-        private LoginFailure(String msg) {
+        public LoginFailure(String msg) {
             super(msg);
         }
     }

plugin/src/test/java/jenkins/plugins/openstack/PluginTestRule.java

@@ -310,6 +310,13 @@ public JCloudsCloud dummyCloud(SlaveOptions opts, JCloudsSlaveTemplate... templa
         return cloud;
     }
 
+    public JCloudsCloud unavailableDummyCloud(SlaveOptions opts, JCloudsSlaveTemplate... templates) {
+        JCloudsCloud cloud = new MockJCloudsCloud(opts,false,templates);
+
+        jenkins.clouds.add(cloud);
+        return cloud;
+    }
+
     public JCloudsCloud configureSlaveLaunchingWithFloatingIP(String labels) {
         return configureSlaveLaunchingWithFloatingIP(dummyCloud(dummySlaveTemplate(labels)));
     }
@@ -642,9 +649,18 @@ public MockJCloudsCloud(SlaveOptions opts, JCloudsSlaveTemplate... templates) {
             super("openstack", "endPointUrl", false,"zone", opts, Arrays.asList(templates), "credentialsId");
         }
 
+        public MockJCloudsCloud(SlaveOptions opts,boolean available, JCloudsSlaveTemplate... templates) {
+            super("openstack", "endPointUrl", false,"zone", opts, Arrays.asList(templates), null);
+        }
+
         @Override
         public @Nonnull Openstack getOpenstack() {
-            return os;
+            if ( getCredentialsId() != null){
+                return os ;
+            } else {
+                throw new LoginFailure("Login failure");
+            }
+
         }
 
         @Override

plugin/src/test/java/jenkins/plugins/openstack/compute/ProvisioningTest.java

@@ -32,6 +32,7 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -439,6 +440,99 @@ public void failIfNoAccessIpFound() {
         }
     }
 
+    @Test
+    public void provisionWhenSharedLabel() throws Exception {
+
+        // Test 1 - premiere assertion 
+        /* 	○ Test1: Everything fine 
+			§ Two clouds configured and up
+			§ Two jobs - excessWorkload = 2 
+			§ Should work fine 
+         */
+
+        SlaveOptions init = j.defaultSlaveOptions();
+        JCloudsSlaveTemplate template1 = j.dummySlaveTemplate(init.getBuilder().instanceCap(1).build(), "generic");
+        JCloudsSlaveTemplate template2 = j.dummySlaveTemplate(init.getBuilder().instanceCap(1).build(), "generic");
+        JCloudsCloud cloud = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template1);
+        JCloudsCloud cloud2 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template2);
+
+        Label generic = Label.get("generic");
+
+        // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628)
+        List<JCloudsCloud> clouds = new ArrayList<JCloudsCloud>();
+        clouds.add(cloud); clouds.add(cloud2); 
+        int jobsCount = 2; 
+        // Until there are no more jobs to build
+        while(jobsCount>0){
+            // try provisioning from the clouds 
+            for (JCloudsCloud c : clouds){
+                if (c.canProvision(generic)){
+                    // update the number of remaining jobs to build
+                    Collection<NodeProvisioner.PlannedNode> plannedNodeList =  c.provision(generic,jobsCount);
+                    jobsCount -= plannedNodeList.size(); 
+                }
+            }  
+        }
+        assertEquals(0,jobsCount);
+
+          // Test 2 - second assertion 
+        /* ○ Test2: First Cloud down 
+			§ First Cloud down - simulate outage by providing invalid credentials 
+			§ Two jobs
+			§ Should work fine - It should build both jobs using the remaining cloud
+         */
+        clouds.clear();
+        cloud = cloud2 = null;
+        JCloudsCloud cloud3 = j.unavailableDummyCloud(init.getBuilder().instanceCap(1).build(), template1);
+        JCloudsCloud cloud4 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template2);
+
+
+
+        // // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628)
+
+        clouds.add(cloud3); clouds.add(cloud4); 
+        int jobsCount2 = 2; 
+        // // Until there are no more jobs to build
+        while(jobsCount2>0){
+            // try provisioning from the clouds 
+            for (JCloudsCloud c : clouds){
+                if (c.canProvision(generic)){
+                    // update the number of remaining jobs to build
+                    jobsCount2 -=  c.provision(generic,jobsCount2).size();
+                }
+            }  
+        }
+
+        assertEquals(0,jobsCount2);
+         // Test 3 - third assertion 
+        /* ○ Test 3: second Cloud down 
+			§ First Cloud down - simulate outage by providing invalid credentials 
+			§ Two jobs
+			§ Should work fine - It should build both jobs using the remaining cloud
+         */
+        clouds.clear();
+        cloud3 = cloud4 = null;
+        JCloudsCloud cloud5 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template1);
+        JCloudsCloud cloud6 = j.unavailableDummyCloud(init.getBuilder().instanceCap(1).build(), template2);
+
+        // // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628)
+        clouds.add(cloud5); clouds.add(cloud6); 
+         int jobsCount3 = 2; 
+        // // Until there are no more jobs to build
+        while(jobsCount3>0){
+            // try provisioning from the clouds 
+            for (JCloudsCloud c : clouds){
+                if (c.canProvision(generic)){
+                    // update the number of remaining jobs to build
+                    jobsCount3 -= c.provision(generic,jobsCount3).size();
+                }
+            }  
+        }
+
+        assertEquals(0,jobsCount3);  
+    }
+
+
     private void verifyPreferredAddressUsed(String expectedAddress, Collection<NetworkAddress> addresses) throws Exception {
         CloudStatistics cs = CloudStatistics.get();
         assertThat(cs.getActivities(), Matchers.iterableWithSize(0));