Further improvements to CKS

api/src/main/java/com/cloud/network/vpc/Vpc.java

@@ -105,4 +105,6 @@ public enum State {
     String getIp6Dns1();
 
     String getIp6Dns2();
+
+    boolean useRouterIpAsResolver();
 }

api/src/main/java/com/cloud/network/vpc/VpcService.java

@@ -48,17 +48,17 @@ public interface VpcService {
      * @param vpcName
      * @param displayText
      * @param cidr
-     * @param networkDomain TODO
+     * @param networkDomain   TODO
      * @param ip4Dns1
      * @param ip4Dns2
-     * @param displayVpc TODO
+     * @param displayVpc      TODO
+     * @param useVrIpResolver
      * @return
      * @throws ResourceAllocationException TODO
      */
     Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain,
                   String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize,
-                  Long asNumber, List<Long> bgpPeerIds)
-            throws ResourceAllocationException;
+                  Long asNumber, List<Long> bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException;
 
     /**
      * Persists VPC record in the database

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -548,6 +548,7 @@ public class ApiConstants {
     public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist";
     public static final String USER_SECRET_KEY = "usersecretkey";
     public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
+    public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver";
     public static final String UPDATE_IN_SEQUENCE = "updateinsequence";
     public static final String VALUE = "value";
     public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";

api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.vpc;
 
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.cloudstack.acl.RoleType;
@@ -125,6 +126,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd {
     @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the VPC tiers")
     private Long asNumber;
 
+    @Parameter(name=ApiConstants.USE_VIRTUAL_ROUTER_IP_RESOLVER, type=CommandType.BOOLEAN,
+            description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2")
+    private Boolean useVrIpResolver;
+
     // ///////////////////////////////////////////////////
     // ///////////////// Accessors ///////////////////////
     // ///////////////////////////////////////////////////
@@ -205,6 +210,10 @@ public Long getAsNumber() {
         return asNumber;
     }
 
+    public boolean getUseVrIpResolver() {
+        return BooleanUtils.toBoolean(useVrIpResolver);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

debian/rules

@@ -70,6 +70,7 @@ override_dh_auto_install:
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm
+	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf
 	mkdir $(DESTDIR)/var/log/$(PACKAGE)/management
 	mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management
 	mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator
@@ -83,6 +84,7 @@ override_dh_auto_install:
 	cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar
 	cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/
 	cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/
+	cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/
 	rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt
 
 	# Bundle cmk in cloudstack-management
@@ -95,6 +97,12 @@ override_dh_auto_install:
 	chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
 
 	install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses
+
+	install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml
+	install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml
+	install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml
+	install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml
+
 	# Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF
 	# This should all be in /etc/cloudstack/management
 	ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf

engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java

@@ -105,6 +105,9 @@ public class VpcVO implements Vpc {
     @Column(name = "ip6Dns2")
     String ip6Dns2;
 
+    @Column(name = "use_router_ip_resolver")
+    boolean useRouterIpResolver = false;
+
     @Transient
     boolean rollingRestart = false;
 
@@ -309,4 +312,13 @@ public String getIp6Dns1() {
     public String getIp6Dns2() {
         return ip6Dns2;
     }
+
+    @Override
+    public boolean useRouterIpAsResolver() {
+        return useRouterIpResolver;
+    }
+
+    public void setUseRouterIpResolver(boolean useRouterIpResolver) {
+        this.useRouterIpResolver = useRouterIpResolver;
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql

@@ -68,6 +68,9 @@ ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_templa
 
 -- Add for_cks column to the user_data table to represent CNI Configuration stored as userdata
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the userdata represent CNI configuration meant for CKS use only"');
+
+-- Add use VR IP as resolver option on VPC
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc','use_router_ip_resolver', 'tinyint(1) DEFAULT 0 COMMENT "use router ip as resolver instead of dns options"');
 -----------------------------------------------------------
 -- END - CKS Enhancements
 -----------------------------------------------------------

packaging/el8/cloud.spec

@@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d
@@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL
 chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk
 
 cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
-
+cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
 cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
 cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp
 cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/
@@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
 #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
 install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management
 
+install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml
+install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml
+install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml
+install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml
+
 # SystemVM template
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
 cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
@@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %attr(0755,root,root) %{_bindir}/%{name}-sysvmadm
 %attr(0755,root,root) %{_bindir}/%{name}-setup-encryption
 %attr(0755,root,root) %{_bindir}/cmk
+%{_datadir}/%{name}-management/cks/conf/*.yml
 %{_datadir}/%{name}-management/setup/*.sql
 %{_datadir}/%{name}-management/setup/*.sh
 %{_datadir}/%{name}-management/setup/server-setup.xml

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -21,6 +21,9 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -143,6 +146,7 @@ public class KubernetesClusterActionWorker {
 
     public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
     public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes";
+    public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks";
 
     protected Logger logger = LogManager.getLogger(getClass());
 
@@ -264,6 +268,11 @@ protected String readResourceFile(String resource) throws IOException {
         return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset());
     }
 
+    protected String readK8sConfigFile(String resource) throws IOException {
+        Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource));
+        return Files.readString(path);
+    }
+
     protected String getControlNodeLoginUser() {
         List<KubernetesClusterVmMapVO> vmMapVOList = getKubernetesClusterVMMaps();
         if (!vmMapVOList.isEmpty()) {
@@ -316,7 +325,7 @@ protected void logMessage(final Level logLevel, final String message, final Exce
     }
 
     protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster,
-        final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
+                                                    final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
         logMessage(logLevel, message, e);
         stateTransitTo(kubernetesCluster.getId(), event);
         detachIsoKubernetesVMs(clusterVMs);
@@ -670,14 +679,14 @@ protected boolean createCloudStackSecret(String[] keys) {
 
         try {
             String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'",
-                scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
+                    scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
             Account account = accountDao.findById(kubernetesCluster.getAccountId());
             if (account != null && account.getType() == Account.Type.PROJECT) {
                 String projectId = projectService.findByProjectAccountId(account.getId()).getUuid();
                 command = String.format("%s -p '%s'", command, projectId);
             }
             Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
-                pkFile, null, command, 10000, 10000, 60000);
+                    pkFile, null, command, 10000, 10000, 60000);
             return result.first();
         } catch (Exception e) {
             String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
@@ -696,7 +705,7 @@ protected File retrieveScriptFile(String filename) {
             writer.close();
         } catch (IOException e) {
             logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s",
-                kubernetesCluster.getName(), filename), e);
+                    kubernetesCluster.getName(), filename), e);
         }
         return file;
     }
@@ -719,11 +728,11 @@ protected void copyScriptFile(String nodeAddress, final int sshPort, File file,
                 sshKeyFile = getManagementServerSshPublicKeyFile();
             }
             SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
-                "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
+                    "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
             // Ensure destination dir scriptPath exists and copy file to destination
             String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination);
             SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
-                cmdStr, 10000, 10000, 10 * 60 * 1000);
+                    cmdStr, 10000, 10000, 10 * 60 * 1000);
         } catch (Exception e) {
             throw new CloudRuntimeException(e);
         }
@@ -771,7 +780,7 @@ protected boolean deployProvider() {
         // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
         if (manager.isDirectAccess(network)) {
             logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
-                kubernetesCluster.getName()), null);
+                    kubernetesCluster.getName()), null);
             return true;
         }
         File pkFile = getManagementServerSshPublicKeyFile();
@@ -782,7 +791,7 @@ protected boolean deployProvider() {
         try {
             String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename);
             Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
-                pkFile, null, command, 10000, 10000, 60000);
+                    pkFile, null, command, 10000, 10000, 60000);
 
             // Maybe the file isn't present. Try and copy it
             if (!result.first()) {
@@ -792,12 +801,12 @@ protected boolean deployProvider() {
 
                 if (!createCloudStackSecret(keys)) {
                     logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
-                        kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+                            kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
                 }
 
                 // If at first you don't succeed ...
                 result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
-                    pkFile, null, command, 10000, 10000, 60000);
+                        pkFile, null, command, 10000, 10000, 60000);
                 if (!result.first()) {
                     throw new CloudRuntimeException(result.second());
                 }
@@ -871,7 +880,7 @@ protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network n
     }
 
     public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException {
-        String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
+        String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml");
         final String sshPubKey = "{{ k8s.ssh.pub.key }}";
         final String joinIpKey = "{{ k8s_control_node.join_ip }}";
         final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java

@@ -41,6 +41,7 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -95,7 +96,7 @@ private boolean removeNodesFromCluster(List<Long> nodeIds, Network network, IpAd
                 continue;
             }
             try {
-                removeNodeVmFromCluster(nodeId, vm.getDisplayName(), publicIp.getAddress().addr());
+                removeNodeVmFromCluster(nodeId, vm.getDisplayName().toLowerCase(Locale.ROOT), publicIp.getAddress().addr());
                 result &= removeNodePortForwardingRules(nodeId, network, vm);
                 if (System.currentTimeMillis() > removeNodeTimeoutTime) {
                     logger.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName()));