diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java index 57fa498e6b1..bec4cd8744c 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java @@ -115,7 +115,9 @@ import io.subutai.core.environment.impl.workflow.modification.SshKeyAdditionWorkflow; import io.subutai.core.environment.impl.workflow.modification.SshKeyRemovalWorkflow; import io.subutai.core.environment.impl.xpeer.RemoteEnvironment; +import io.subutai.core.hostregistry.api.HostDisconnectedException; import io.subutai.core.hostregistry.api.HostListener; +import io.subutai.core.hostregistry.api.HostRegistry; import io.subutai.core.identity.api.IdentityManager; import io.subutai.core.identity.api.model.Session; import io.subutai.core.identity.api.model.User; @@ -2276,19 +2278,29 @@ public Void run() } } - //TODO need to remember if env got unhealthy during key exchange - protected void doResetP2Pkeys() { try { //process only SS side environments - for ( Environment environment : environmentService.getAll() ) + for ( LocalEnvironment environment : environmentService.getAll() ) { - if ( !( ( environment.getStatus() != EnvironmentStatus.HEALTHY && !allContainersAreRunning( - environment ) ) || ( ( System.currentTimeMillis() - environment.getCreationTimestamp() ) - < TimeUnit.MINUTES.toMillis( RESET_ENVS_P2P_KEYS_INTERVAL_MIN ) ) ) ) + if ( + //process only healthy environments + ( environment.getStatus() == EnvironmentStatus.HEALTHY || + + //or environments unhealthy due to p2p key reset + ( environment.getStatus() == EnvironmentStatus.UNHEALTHY && Objects + .equals( environment.getStatusDescription(), + P2PSecretKeyModificationWorkflow.P2P_CAUSE ) ) + + ) && + + allContainersAreRunning( environment ) && + + ( System.currentTimeMillis() - environment.getCreationTimestamp() ) >= TimeUnit.MINUTES + .toMillis( RESET_ENVS_P2P_KEYS_INTERVAL_MIN ) ) { final String secretKey = UUID.randomUUID().toString(); final long keyTtl = Common.DEFAULT_P2P_SECRET_KEY_TTL_SEC; @@ -2305,9 +2317,24 @@ protected void doResetP2Pkeys() private boolean allContainersAreRunning( final Environment environment ) { + if ( CollectionUtil.isCollectionEmpty( environment.getContainerHosts() ) ) + { + return false; + } + for ( ContainerHost containerHost : environment.getContainerHosts() ) { - if ( containerHost.getState() != ContainerHostState.RUNNING ) + try + { + ContainerHostInfo containerHostInfo = + getHostRegistry().getContainerHostInfoById( containerHost.getId() ); + + if ( containerHostInfo.getState() != ContainerHostState.RUNNING ) + { + return false; + } + } + catch ( HostDisconnectedException ignore ) { return false; } @@ -2317,6 +2344,12 @@ private boolean allContainersAreRunning( final Environment environment ) } + protected HostRegistry getHostRegistry() + { + return ServiceLocator.lookup( HostRegistry.class ); + } + + private void syncEnvironments() { if ( System.currentTimeMillis() - lastEnvSyncTs >= TimeUnit.MINUTES diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java index e8ee357c441..6cf2acf60dc 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/entity/LocalEnvironment.java @@ -136,6 +136,10 @@ public class LocalEnvironment implements Environment, Serializable @JsonProperty( "status" ) private EnvironmentStatus status = EnvironmentStatus.EMPTY; + @JsonIgnore + private String statusDescription; + + @Column( name = "initial_blueprint" ) @JsonIgnore @Lob @@ -346,6 +350,18 @@ public void setStatus( EnvironmentStatus status ) } + public String getStatusDescription() + { + return statusDescription; + } + + + public void setStatusDescription( final String statusDescription ) + { + this.statusDescription = statusDescription; + } + + String getRawTopology() { return rawBlueprint; diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java index 2396be3e544..7470e71fe74 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/workflow/modification/P2PSecretKeyModificationWorkflow.java @@ -13,6 +13,7 @@ public class P2PSecretKeyModificationWorkflow extends CancellableWorkflow { + public static final String P2P_CAUSE = "P2P key reset failed"; private LocalEnvironment environment; private final String p2pSecretKey; private final long p2pSecretKeyTtlSeconds; @@ -97,6 +98,8 @@ public void fail( final String message, final Throwable e ) { environment.setStatus( EnvironmentStatus.UNHEALTHY ); + environment.setStatusDescription( P2P_CAUSE ); + saveEnvironment(); operationTracker.addLogFailed( message ); diff --git a/management/server/core/environment-manager/environment-manager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java b/management/server/core/environment-manager/environment-manager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java index 3a3ef4a7641..7c99db69a47 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/test/java/io/subutai/core/environment/impl/EnvironmentManagerImplTest.java @@ -34,6 +34,8 @@ import io.subutai.common.environment.Node; import io.subutai.common.environment.Nodes; import io.subutai.common.environment.Topology; +import io.subutai.common.host.ContainerHostInfo; +import io.subutai.common.host.ContainerHostState; import io.subutai.common.metric.Alert; import io.subutai.common.metric.AlertValue; import io.subutai.common.network.ReservedNetworkResources; @@ -74,6 +76,7 @@ import io.subutai.core.environment.impl.workflow.modification.P2PSecretKeyModificationWorkflow; import io.subutai.core.environment.impl.workflow.modification.SshKeyAdditionWorkflow; import io.subutai.core.environment.impl.workflow.modification.SshKeyRemovalWorkflow; +import io.subutai.core.hostregistry.api.HostRegistry; import io.subutai.core.identity.api.IdentityManager; import io.subutai.core.identity.api.model.Session; import io.subutai.core.identity.api.model.User; @@ -182,6 +185,11 @@ public class EnvironmentManagerImplTest @Mock SystemManager systemManager; + @Mock + HostRegistry hostRegistry; + @Mock + ContainerHostInfo containerHostInfo; + class EnvironmentManagerImplSUT extends EnvironmentManagerImpl { @@ -1250,6 +1258,11 @@ public void testResetP2pKey() throws Exception { doNothing().when( environmentManager ).resetP2PSecretKey( anyString(), anyString(), anyLong(), anyBoolean() ); doReturn( EnvironmentStatus.HEALTHY ).when( environment ).getStatus(); + doReturn( Sets.newHashSet( environmentContainer ) ).when( environment ).getContainerHosts(); + + doReturn( hostRegistry ).when( environmentManager ).getHostRegistry(); + doReturn( containerHostInfo ).when( hostRegistry ).getContainerHostInfoById( anyString() ); + doReturn( ContainerHostState.RUNNING ).when( containerHostInfo ).getState(); environmentManager.doResetP2Pkeys();