Merge pull request #2767 from subutai-io/dev

#2765 Dev -> Master
subutai-io · Dec 19, 2018 · 49048cc · 49048cc
2 parents 9e006b1 + d1edb93
commit 49048cc
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 7 deletions.
diff --git a/...t-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java b/...t-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java
@@ -115,7 +115,9 @@
 import io.subutai.core.environment.impl.workflow.modification.SshKeyAdditionWorkflow;
 import io.subutai.core.environment.impl.workflow.modification.SshKeyRemovalWorkflow;
 import io.subutai.core.environment.impl.xpeer.RemoteEnvironment;
+import io.subutai.core.hostregistry.api.HostDisconnectedException;
 import io.subutai.core.hostregistry.api.HostListener;
+import io.subutai.core.hostregistry.api.HostRegistry;
 import io.subutai.core.identity.api.IdentityManager;
 import io.subutai.core.identity.api.model.Session;
 import io.subutai.core.identity.api.model.User;
@@ -2276,19 +2278,29 @@ public Void run()
         }
     }
 
-    //TODO need to remember if env got unhealthy during key exchange
-
 
     protected void doResetP2Pkeys()
     {
         try
         {
             //process only SS side environments
-            for ( Environment environment : environmentService.getAll() )
+            for ( LocalEnvironment environment : environmentService.getAll() )
             {
-                if ( !( ( environment.getStatus() != EnvironmentStatus.HEALTHY && !allContainersAreRunning(
-                        environment ) ) || ( ( System.currentTimeMillis() - environment.getCreationTimestamp() )
-                        < TimeUnit.MINUTES.toMillis( RESET_ENVS_P2P_KEYS_INTERVAL_MIN ) ) ) )
+                if (
+                    //process only healthy environments
+                        ( environment.getStatus() == EnvironmentStatus.HEALTHY ||
+
+                                //or environments unhealthy due to p2p key reset
+                                ( environment.getStatus() == EnvironmentStatus.UNHEALTHY && Objects
+                                        .equals( environment.getStatusDescription(),
+                                                P2PSecretKeyModificationWorkflow.P2P_CAUSE ) )
+
+                        ) &&
+
+                                allContainersAreRunning( environment ) &&
+
+                                ( System.currentTimeMillis() - environment.getCreationTimestamp() ) >= TimeUnit.MINUTES
+                                        .toMillis( RESET_ENVS_P2P_KEYS_INTERVAL_MIN ) )
                 {
                     final String secretKey = UUID.randomUUID().toString();
                     final long keyTtl = Common.DEFAULT_P2P_SECRET_KEY_TTL_SEC;
@@ -2305,9 +2317,24 @@ protected void doResetP2Pkeys()
 
     private boolean allContainersAreRunning( final Environment environment )
     {
+        if ( CollectionUtil.isCollectionEmpty( environment.getContainerHosts() ) )
+        {
+            return false;
+        }
+
         for ( ContainerHost containerHost : environment.getContainerHosts() )
         {
-            if ( containerHost.getState() != ContainerHostState.RUNNING )
+            try
+            {
+                ContainerHostInfo containerHostInfo =
+                        getHostRegistry().getContainerHostInfoById( containerHost.getId() );
+
+                if ( containerHostInfo.getState() != ContainerHostState.RUNNING )
+                {
+                    return false;
+                }
+            }
+            catch ( HostDisconnectedException ignore )
             {
                 return false;
             }
@@ -2317,6 +2344,12 @@ private boolean allContainersAreRunning( final Environment environment )
     }
 
 
+    protected HostRegistry getHostRegistry()
+    {
+        return ServiceLocator.lookup( HostRegistry.class );
+    }
+
+
     private void syncEnvironments()
     {
         if ( System.currentTimeMillis() - lastEnvSyncTs >= TimeUnit.MINUTES

diff --git a/...-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java b/...-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java
@@ -136,6 +136,10 @@ public class LocalEnvironment implements Environment, Serializable
     @JsonProperty( "status" )
     private EnvironmentStatus status = EnvironmentStatus.EMPTY;
 
+    @JsonIgnore
+    private String statusDescription;
+
+
     @Column( name = "initial_blueprint" )
     @JsonIgnore
     @Lob
@@ -346,6 +350,18 @@ public void setStatus( EnvironmentStatus status )
     }
 
 
+    public String getStatusDescription()
+    {
+        return statusDescription;
+    }
+
+
+    public void setStatusDescription( final String statusDescription )
+    {
+        this.statusDescription = statusDescription;
+    }
+
+
     String getRawTopology()
     {
         return rawBlueprint;

diff --git a/...subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java b/...subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java
@@ -13,6 +13,7 @@
 public class P2PSecretKeyModificationWorkflow
         extends CancellableWorkflow<P2PSecretKeyModificationWorkflow.P2PSecretKeyModificationPhase>
 {
+    public static final String P2P_CAUSE = "P2P key reset failed";
     private LocalEnvironment environment;
     private final String p2pSecretKey;
     private final long p2pSecretKeyTtlSeconds;
@@ -97,6 +98,8 @@ public void fail( final String message, final Throwable e )
     {
         environment.setStatus( EnvironmentStatus.UNHEALTHY );
 
+        environment.setStatusDescription( P2P_CAUSE );
+
         saveEnvironment();
 
         operationTracker.addLogFailed( message );

diff --git a/...nager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java b/...nager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java
@@ -34,6 +34,8 @@
 import io.subutai.common.environment.Node;
 import io.subutai.common.environment.Nodes;
 import io.subutai.common.environment.Topology;
+import io.subutai.common.host.ContainerHostInfo;
+import io.subutai.common.host.ContainerHostState;
 import io.subutai.common.metric.Alert;
 import io.subutai.common.metric.AlertValue;
 import io.subutai.common.network.ReservedNetworkResources;
@@ -74,6 +76,7 @@
 import io.subutai.core.environment.impl.workflow.modification.P2PSecretKeyModificationWorkflow;
 import io.subutai.core.environment.impl.workflow.modification.SshKeyAdditionWorkflow;
 import io.subutai.core.environment.impl.workflow.modification.SshKeyRemovalWorkflow;
+import io.subutai.core.hostregistry.api.HostRegistry;
 import io.subutai.core.identity.api.IdentityManager;
 import io.subutai.core.identity.api.model.Session;
 import io.subutai.core.identity.api.model.User;
@@ -182,6 +185,11 @@ public class EnvironmentManagerImplTest
     @Mock
     SystemManager systemManager;
 
+    @Mock
+    HostRegistry hostRegistry;
+    @Mock
+    ContainerHostInfo containerHostInfo;
+
 
     class EnvironmentManagerImplSUT extends EnvironmentManagerImpl
     {
@@ -1250,6 +1258,11 @@ public void testResetP2pKey() throws Exception
     {
         doNothing().when( environmentManager ).resetP2PSecretKey( anyString(), anyString(), anyLong(), anyBoolean() );
         doReturn( EnvironmentStatus.HEALTHY ).when( environment ).getStatus();
+        doReturn( Sets.newHashSet( environmentContainer ) ).when( environment ).getContainerHosts();
+
+        doReturn( hostRegistry ).when( environmentManager ).getHostRegistry();
+        doReturn( containerHostInfo ).when( hostRegistry ).getContainerHostInfoById( anyString() );
+        doReturn( ContainerHostState.RUNNING ).when( containerHostInfo ).getState();
 
         environmentManager.doResetP2Pkeys();