diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java index c443c3873814..27c33d07928c 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java @@ -122,7 +122,7 @@ public enum ContainerMetrics implements VespaMetrics { QUERIES("queries", Unit.OPERATION, "Query volume"), QUERY_CONTAINER_LATENCY("query_container_latency", Unit.MILLISECOND, "The query execution time consumed in the container"), QUERY_LATENCY("query_latency", Unit.MILLISECOND, "The overall query latency as seen by the container"), - QUERY_TIMEOUT("query_timeout", Unit.MILLISECOND, "The amount of time allowed for query execytion, from the client"), + QUERY_TIMEOUT("query_timeout", Unit.MILLISECOND, "The amount of time allowed for query execution, from the client"), FAILED_QUERIES("failed_queries", Unit.OPERATION, "The number of failed queries"), DEGRADED_QUERIES("degraded_queries", Unit.OPERATION, "The number of degraded queries, e.g. due to some conent nodes not responding in time"), HITS_PER_QUERY("hits_per_query", Unit.HIT_PER_QUERY, "The number of hits returned"), diff --git a/container-core/src/main/java/com/yahoo/processing/request/Properties.java b/container-core/src/main/java/com/yahoo/processing/request/Properties.java index ac43f99472e5..060916a02986 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/Properties.java +++ b/container-core/src/main/java/com/yahoo/processing/request/Properties.java @@ -265,7 +265,7 @@ public final void clearAll(CompoundName name) { /** * Sets all properties having this name as a compound prefix to null. - * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * I.e. clearAll("a") will clear the value of "a" and "a.b" but not "ab". * * @param name the compound prefix of the properties to clear * @throws RuntimeException if no instance in the chain accepted this name-value pair diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index d6671d41cbd6..a06817916503 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -205,7 +205,7 @@ public NodeRepository(NodeFlavors flavors, */ public boolean exclusiveAllocation(ClusterSpec clusterSpec) { return clusterSpec.isExclusive() || - ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || + ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || ( !zone().cloud().allowHostSharing() && !sharedHosts.value().isEnabled(clusterSpec.type().name())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 3af63125474b..23c8934b1089 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -194,27 +194,34 @@ private boolean offeredNodeHasParentHostnameAlreadyAccepted(NodeCandidate candid return false; } + /** + * Returns whether allocating the candidate on this host would violate exclusivity constraints. + * Note that while we currently require that exclusive allocations uses the entire host, + * this method also handles the case where smaller exclusive nodes are allocated on it. + */ private boolean violatesExclusivity(NodeCandidate candidate) { - if (candidate.parentHostname().isEmpty()) return false; - - // In nodes which does not allow host sharing, exclusivity is violated if... - if ( ! nodeRepository.zone().cloud().allowHostSharing()) { - // TODO: Write this in a way that is simple to read - // If either the parent is dedicated to a cluster type different from this cluster - return ! candidate.parent.flatMap(Node::exclusiveToClusterType).map(cluster.type()::equals).orElse(true) || - // or this cluster is requiring exclusivity, but the host is exclusive to a different owner - (requestedNodes.isExclusive() && !candidate.parent.flatMap(Node::exclusiveToApplicationId).map(application::equals).orElse(false)); + if (candidate.parent.isEmpty()) return false; + + if (nodeRepository.exclusiveAllocation(cluster)) { + // Node must allocate the host entirely and not violate application or cluster type constraints + var parent = candidate.parent.get(); + if (!candidate.resources().isUnspecified() && + ! nodeRepository.resourcesCalculator().advertisedResourcesOf(parent.flavor()).compatibleWith(candidate.resources())) return true; + if (parent.exclusiveToApplicationId().isPresent() && !parent.exclusiveToApplicationId().get().equals(application)) return true; + if (parent.exclusiveToClusterType().isPresent() && !parent.exclusiveToClusterType().get().equals(cluster.type())) return true; + return false; } - - // In zones with shared hosts we require that if either of the nodes on the host requires exclusivity, - // then all the nodes on the host must have the same owner - for (Node nodeOnHost : allNodes.childrenOf(candidate.parentHostname().get())) { - if (nodeOnHost.allocation().isEmpty()) continue; - if (requestedNodes.isExclusive() || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { - if ( ! nodeOnHost.allocation().get().owner().equals(application)) return true; + else { + // If any of the nodes on the host requires exclusivity to another application, allocating on it is a violation + for (Node nodeOnHost : allNodes.childrenOf(candidate.parentHostname().get())) { + if (nodeOnHost.allocation().isEmpty()) continue; + if (nodeOnHost.allocation().get().membership().cluster().isExclusive()) { + if (!nodeOnHost.allocation().get().owner().equals(application)) + return true; + } } + return false; } - return false; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 24ea93618233..6efb9b6f4aa9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -47,7 +47,7 @@ public class MockHostProvisioner implements HostProvisioner { private int deprovisionedHosts = 0; private EnumSet behaviours = EnumSet.noneOf(Behaviour.class); - private Map hostFlavors = new HashMap<>(); + private final Map hostFlavors = new HashMap<>(); public MockHostProvisioner(List flavors, MockNameResolver nameResolver, int memoryTaxGb) { this.flavors = List.copyOf(flavors); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0a614cc9b2b7..51eeb8717f5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -97,10 +97,10 @@ public MockNodeRepository(MockCurator curator, NodeFlavors flavors, Zone zone) { defaultCloudAccount = zone.cloud().account(); curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); - populate(); + populate(zone); } - private void populate() { + private void populate(Zone zone) { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider()); List nodes = new ArrayList<>(); @@ -202,16 +202,26 @@ private void populate() { .vespaVersion("6.42") .loadBalancerSettings(new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne")))) .build(); + ClusterResources min, max; + if (zone.system().isPublic()) { // resources must match one of the flavors used in this mock ("large"), since this is a container cluster + min = new ClusterResources(2, 1, new NodeResources(4, 32, 1600, 1)); + max = new ClusterResources(8, 2, new NodeResources(8, 64, 3200, 1)); + } + else { // resources must fit on actually provisioned hosts + min = new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)); + max = new ClusterResources(8, 2, new NodeResources(4, 16, 1000, 1)); + } + activate(provisioner.prepare(app1Id, cluster1Id, - Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)), - new ClusterResources(8, 2, new NodeResources(4, 16, 1000, 1)), + Capacity.from(min, + max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()), - null), app1Id, provisioner); + null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); cluster1 = cluster1.withSuggested(new Autoscaling(Autoscaling.Status.unavailable, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 81947251e64e..a8be7ac3af4d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -50,4 +50,5 @@ public Optional getHostProvisioner() { public HostResourcesCalculator getHostResourcesCalculator() { return hostResourcesCalculator; } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 0a1bdb7b400f..95291e821bc4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -376,8 +376,8 @@ private void replace_config_server_like(NodeType hostType) { // Provision config servers for (int i = 0; i < provisionedHosts.size(); i++) { - tester.makeReadyChildren(1, i + 1, new NodeResources(1.5, 8, 50, 0.3), hostType.childNodeType(), - provisionedHosts.get(i).hostname(), (nodeIndex) -> "cfg" + nodeIndex); + tester.makeReadyChildren(1, i + 1, new NodeResources(1.0, 30, 20, 0.3), hostType.childNodeType(), + provisionedHosts.get(i).hostname(), (nodeIndex) -> "cfg" + nodeIndex); } tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 382d28403777..95b7303bf8a3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.Architecture; @@ -28,9 +29,11 @@ import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -51,6 +54,25 @@ public class DynamicProvisioningTest { private final MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup(); + @Test + public void test_provisioning_containers_wont_use_shared_hosts_on_public() { + var resources = new NodeResources(2.7, 9, 17, 0.1); + var now = new ClusterResources(2, 1, resources); + try { + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.container) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(now)) + .hostCount(2) + .build(); + } + catch (NodeAllocationException e) { + assertTrue("Contains 'No host flavor matches': " + e.getMessage(), + e.getMessage().contains("No host flavor matches")); + } + } + @Test public void dynamically_provision_with_empty_node_repo() { var tester = tester(true); @@ -117,7 +139,7 @@ public void in_place_resize_not_allowed_on_exclusive_to_hosts() { @Test public void avoids_allocating_to_empty_hosts() { - var tester = tester(false); + var tester = tester(true); tester.makeReadyHosts(6, new NodeResources(12, 12, 200, 12)); tester.activateTenantHosts(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d0ff11fde0c5..bfe176bc4296 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -90,16 +90,16 @@ public class ProvisioningTester { private int nextIP = 0; private ProvisioningTester(Curator curator, - NodeFlavors nodeFlavors, - HostResourcesCalculator resourcesCalculator, - Zone zone, - NameResolver nameResolver, - DockerImage containerImage, - Orchestrator orchestrator, - HostProvisioner hostProvisioner, - LoadBalancerServiceMock loadBalancerService, - FlagSource flagSource, - int spareCount) { + NodeFlavors nodeFlavors, + HostResourcesCalculator resourcesCalculator, + Zone zone, + NameResolver nameResolver, + DockerImage containerImage, + Orchestrator orchestrator, + HostProvisioner hostProvisioner, + LoadBalancerServiceMock loadBalancerService, + FlagSource flagSource, + int spareCount) { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 0744d82c85b3..91f20ff8ee55 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -364,12 +364,12 @@ public void application_deployment_with_exclusive_app_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, hostResources).activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId("app1"); - prepareAndActivate(application1, 2, true, nodeResources, tester); + prepareAndActivate(application1, ClusterSpec.Type.content, 2, true, hostResources, tester); assertEquals(Set.of("host-1.yahoo.com", "host-2.yahoo.com"), hostsOf(tester.getNodes(application1, Node.State.active))); ApplicationId application2 = ProvisioningTester.applicationId("app2"); - prepareAndActivate(application2, 2, false, nodeResources, tester); + prepareAndActivate(application2, ClusterSpec.Type.content, 2, false, nodeResources, tester); assertEquals("Application is assigned to separate hosts", Set.of("host-3.yahoo.com", "host-4.yahoo.com"), hostsOf(tester.getNodes(application2, Node.State.active))); @@ -379,16 +379,15 @@ public void application_deployment_with_exclusive_app_first() { @Test public void application_deployment_with_exclusive_app_last() { NodeResources hostResources = new NodeResources(10, 40, 1000, 10); - NodeResources nodeResources = new NodeResources(2, 4, 100, 1); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, hostResources).activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId("app1"); - prepareAndActivate(application1, 2, false, nodeResources, tester); + prepareAndActivate(application1, 2, false, hostResources, tester); assertEquals(Set.of("host-1.yahoo.com", "host-2.yahoo.com"), hostsOf(tester.getNodes(application1, Node.State.active))); ApplicationId application2 = ProvisioningTester.applicationId("app2"); - prepareAndActivate(application2, 2, true, nodeResources, tester); + prepareAndActivate(application2, ClusterSpec.Type.content, 2, true, hostResources, tester); assertEquals("Application is assigned to separate hosts", Set.of("host-3.yahoo.com", "host-4.yahoo.com"), hostsOf(tester.getNodes(application2, Node.State.active))); @@ -398,59 +397,25 @@ public void application_deployment_with_exclusive_app_last() { @Test public void application_deployment_change_to_exclusive_and_back() { NodeResources hostResources = new NodeResources(10, 40, 1000, 10); - NodeResources nodeResources = new NodeResources(2, 4, 100, 1); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, hostResources).activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId(); - prepareAndActivate(application1, 2, false, nodeResources, tester); + prepareAndActivate(application1, ClusterSpec.Type.content, 2, false, hostResources, tester); for (Node node : tester.getNodes(application1, Node.State.active)) assertFalse(node.allocation().get().membership().cluster().isExclusive()); - prepareAndActivate(application1, 2, true, nodeResources, tester); + prepareAndActivate(application1, ClusterSpec.Type.content, 2, true, hostResources, tester); assertEquals(Set.of("host-1.yahoo.com", "host-2.yahoo.com"), hostsOf(tester.getNodes(application1, Node.State.active))); for (Node node : tester.getNodes(application1, Node.State.active)) assertTrue(node.allocation().get().membership().cluster().isExclusive()); - prepareAndActivate(application1, 2, false, nodeResources, tester); + prepareAndActivate(application1, ClusterSpec.Type.content, 2, false, hostResources, tester); assertEquals(Set.of("host-1.yahoo.com", "host-2.yahoo.com"), hostsOf(tester.getNodes(application1, Node.State.active))); for (Node node : tester.getNodes(application1, Node.State.active)) assertFalse(node.allocation().get().membership().cluster().isExclusive()); } - /** Non-exclusive app first, then an exclusive: Should give the same result as above */ - @Test - public void application_deployment_with_exclusive_app_causing_allocation_failure() { - ApplicationId application1 = ApplicationId.from("tenant1", "app1", "default"); - ApplicationId application2 = ApplicationId.from("tenant2", "app2", "default"); - ApplicationId application3 = ApplicationId.from("tenant1", "app3", "default"); - NodeResources hostResources = new NodeResources(10, 40, 1000, 10); - NodeResources nodeResources = new NodeResources(2, 4, 100, 1); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, hostResources).activateTenantHosts(); - - prepareAndActivate(application1, 2, true, nodeResources, tester); - assertEquals(Set.of("host-1.yahoo.com", "host-2.yahoo.com"), - hostsOf(tester.getNodes(application1, Node.State.active))); - - try { - prepareAndActivate(application2, 3, false, nodeResources, tester); - fail("Expected allocation failure"); - } - catch (Exception e) { - assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", - "Could not satisfy request for 3 nodes with " + - "[vcpu: 2.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] " + - "in tenant2.app2 container cluster 'my-container' 6.39: " + - "Node allocation failure on group 0: " + - "Not enough suitable nodes available due to host exclusivity constraints", - e.getMessage()); - } - - // Adding 3 nodes of another application for the same tenant works - prepareAndActivate(application3, 2, true, nodeResources, tester); - } - @Test public void storage_type_must_match() { try { @@ -667,8 +632,12 @@ private void assertDistinctParentHosts(NodeList nodes, ClusterSpec.Type clusterT } private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, NodeResources resources, ProvisioningTester tester) { + prepareAndActivate(application, ClusterSpec.Type.container, nodeCount, exclusive, resources, tester); + } + + private void prepareAndActivate(ApplicationId application, ClusterSpec.Type clusterType, int nodeCount, boolean exclusive, NodeResources resources, ProvisioningTester tester) { Set hosts = new HashSet<>(tester.prepare(application, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("my-container")).vespaVersion("6.39").exclusive(exclusive).build(), + ClusterSpec.request(clusterType, ClusterSpec.Id.from("my-container")).vespaVersion("6.39").exclusive(exclusive).build(), Capacity.from(new ClusterResources(nodeCount, 1, resources), false, true))); tester.activate(application, hosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java index 7fe2d77b6471..b03cd26243e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -16,7 +16,7 @@ * * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. * - * @author bratseth + * @author freva */ public class ArchiveApiTest { @@ -55,7 +55,6 @@ public void archive_uris() throws IOException { tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "archiveUri", false); } - private void assertFile(Request request, String file) throws IOException { tester.assertFile(request, file); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java index 729b6b813cd2..3b9715757770 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -22,7 +22,8 @@ public void createTester() { @After public void closeTester() { - tester.close(); + if (tester != null) + tester.close(); } @Test