improve start script

Signed-off-by: duanmengkk <[email protected]>

Makefile

@@ -19,7 +19,7 @@ ifeq ($(GOOS), darwin)
 	TARGETS := $(MACOS_TARGETS)
 endif
 
-CTL_TARGETS := nodectl
+#CTL_TARGETS := nestctl
 
 # Build code.
 #

cluster/images/agent.Dockerfile

@@ -1,4 +1,4 @@
-FROM m.daocloud.io/docker.io/ubuntu AS release-env
+FROM docker.io/ubuntu AS release-env
 
 ARG BINARY
 
@@ -17,7 +17,7 @@ RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.aliyun.com/ubu
     sed -i 's|http://archive.ubuntu.com/ubuntu/|http://mirrors.aliyun.com/ubuntu/|' /etc/apt/sources.list
 
 RUN apt-get update && \
-    apt-get install -y rsync pwgen
+    apt-get install -y rsync pwgen sudo
 
 COPY ${BINARY} /app
 

cluster/images/buildx.agent.Dockerfile

@@ -1,4 +1,4 @@
-FROM m.daocloud.io/docker.io/ubuntu AS release-env
+FROM docker.io/ubuntu AS release-env
 
 ARG BINARY
 ARG TARGETPLATFORM
@@ -17,7 +17,7 @@ RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.aliyun.com/ubu
     sed -i 's|http://archive.ubuntu.com/ubuntu/|http://mirrors.aliyun.com/ubuntu/|' /etc/apt/sources.list
 
 RUN apt-get update && \
-    apt-get install -y rsync pwgen
+    apt-get install -y rsync pwgen sudo
 
 COPY ${TARGETPLATFORM}/${BINARY} /app
 

deploy/node-agent.yml → deploy/node-agent.yaml

@@ -1,3 +1,38 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kosmos-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubenest-node-agent
+  namespace: kosmos-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubenest-node-agent
+rules:
+  - apiGroups: ['*']
+    resources: ['*']
+    verbs: ["*"]
+  - nonResourceURLs: ['*']
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubenest-node-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubenest-node-agent
+subjects:
+  - kind: ServiceAccount
+    name: kubenest-node-agent
+    namespace: kosmos-system
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -15,6 +50,7 @@ spec:
       hostPID: true # access host pid
       hostIPC: true # access host ipc
       hostNetwork: true # access host network
+      serviceAccountName: kubenest-node-agent
       tolerations:
         - operator: Exists # run on all nodes
       initContainers:
@@ -70,4 +106,14 @@ spec:
         - name: systemd-path
           hostPath:
             path: /etc/systemd/system
-            type: DirectoryOrCreate
+            type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: node-agent-secret
+  namespace: kosmos-system
+type: kubernetes.io/basic-auth
+data:
+  username: {{ .USERNAME }}
+  password: {{ .PASSWORD }}

hack/cluster.sh

@@ -13,7 +13,7 @@ REUSE=${REUSE:-false}
 USE_LOCAL_ARTIFACTS=${USE_LOCAL_ARTIFACTS:-true}
 VERSION=${VERSION:-latest}
 
-CN_ZONE=${CN_ZONE:-true}
+CN_ZONE=${CN_ZONE:-false}
 source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
 
 # default cert and key for node server https
@@ -122,18 +122,18 @@ function prepare_docker_image() {
 
   if [ "${CN_ZONE}" == false ]; then
     # 使用 Calico 的官方镜像源
-    local calico_prefix="calico"
+    local calico_prefix=""
     local operator_prefix="quay.io"
   else
     # 使用 DaoCloud 镜像源
-    calico_prefix="docker.m.daocloud.io"
+    calico_prefix="docker.m.daocloud.io/"
     operator_prefix="quay.m.daocloud.io"
   fi
 
   # 拉取和标记 Calico 镜像
   for image in "${calico_images[@]}"; do
-    docker pull "${calico_prefix}/${image}:${version}"
-    docker tag "${calico_prefix}/${image}:${version}" "${image}:${version}"
+    docker pull "${calico_prefix}${image}:${version}"
+    docker tag "${calico_prefix}${image}:${version}" "${image}:${version}"
   done
 
   # 拉取和标记 Operator 镜像
@@ -220,131 +220,32 @@ function create_cluster() {
   echo "all node ready"
 }
 
-function join_cluster() {
-  local host_cluster=$1
-  local member_cluster=$2
-  local kubeconfig_path="${ROOT}/environments/${member_cluster}/kubeconfig"
-  local hostConfig_path="${ROOT}/environments/${host_cluster}/kubeconfig"
-  local base64_kubeconfig=$(util::get_base64_kubeconfig <"$kubeconfig_path")
-  echo " base64 kubeconfig successfully converted: $base64_kubeconfig "
-
-  local common_metadata=""
-  if [ "$host_cluster" == "$member_cluster" ]; then
-    common_metadata="annotations:
-    kosmos.io/cluster-role: root"
-  fi
-
-  cat <<EOF | kubectl --kubeconfig "${hostConfig_path}" apply -f -
-apiVersion: kosmos.io/v1alpha1
-kind: Cluster
-metadata:
-  $common_metadata
-  name: ${member_cluster}
-spec:
-  imageRepository: "ghcr.io/kosmos-io"
-  kubeconfig: "$base64_kubeconfig"
-  clusterLinkOptions:
-    cni: "calico"
-    ipFamily: ipv4
-    defaultNICName: eth0
-    networkType: "gateway"
-  clusterTreeOptions:
-    enable: true
-EOF
-  kubectl --kubeconfig "${hostConfig_path}" apply -f "$ROOT"/deploy/clusterlink-namespace.yml
-  kubectl --kubeconfig "${hostConfig_path}" apply -f "$ROOT"/deploy/clusterlink-datapanel-rbac.yml
-}
-
-function join_cluster_by_ctl() {
-  local host_cluster=$1
-  local member_cluster=$2
-  local hostClusterDir=$3
-  local memberClusterDir=$4
-  "${ROOT}"/_output/bin/"$os"/"$arch"/kosmosctl join cluster --name "$member_cluster" --host-kubeconfig "$hostClusterDir/kubeconfig" --kubeconfig "$memberClusterDir/kubeconfig" --inner-kubeconfig "$memberClusterDir/kubeconfig-nodeIp" --enable-all --version ${VERSION}
-}
-
-function addTaint() {
-  local host_cluster=$1
-  local member_cluster=$2
-  leafnode="kosmos-${member_cluster}"
-  HOST_CLUSTER_DIR="${ROOT}/environments/${host_cluster}"
-
-  sleep 100 && kubectl --kubeconfig $HOST_CLUSTER_DIR/kubeconfig get node -owide
-  kubectl --kubeconfig $HOST_CLUSTER_DIR/kubeconfig taint nodes $leafnode test-node/e2e=leafnode:NoSchedule
-}
-
-function deploy_cluster_by_ctl() {
-  local -r clustername=$1
-  local -r kubeconfig=$2
-  local -r innerKubeconfig=$3
-  load_cluster_images "$clustername"
-  CLUSTER_DIR="${ROOT}/environments/${clustername}"
-
-  "${ROOT}"/_output/bin/"$os"/"$arch"/kosmosctl install --version ${VERSION} --kubeconfig "${kubeconfig}" --inner-kubeconfig "${innerKubeconfig}"
-
-  util::wait_for_condition "kosmos ${clustername} clustertree are ready" \
-    "kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system get deploy clustertree-cluster-manager -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \
-    300
-}
-
-function deploy_cluster() {
+function load_kubenetst_cluster_images() {
   local -r clustername=$1
-  CLUSTER_DIR="${ROOT}/environments/${clustername}"
-
-  load_cluster_images "$clustername"
-
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/clusterlink-namespace.yml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/kosmos-rbac.yml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/crds
-  util::wait_for_crd clusternodes.kosmos.io clusters.kosmos.io clusterdistributionpolicies.kosmos.io distributionpolicies.kosmos.io
-
-  sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clusterlink-network-manager.yml" "$ROOT"/deploy/clusterlink-network-manager.yml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clusterlink-network-manager.yml"
-
-  echo "cluster $clustername deploy clusterlink success"
 
-  sed -e "s|__VERSION__|$VERSION|g" -e "s|__CERT__|$CERT|g" -e "s|__KEY__|$KEY|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clustertree-cluster-manager.yml"
-
-  echo "cluster $clustername deploy clustertree success"
-
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system delete secret controlpanel-config || true
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system create secret generic controlpanel-config --from-file=kubeconfig="${ROOT}/environments/cluster-host/kubeconfig"
-  sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clusterlink-operator.yml" "$ROOT"/deploy/clusterlink-operator.yml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clusterlink-operator.yml"
-
-  echo "cluster $clustername deploy clusterlink-operator success"
-
-  sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/kosmos-scheduler.yml" "$ROOT"/deploy/scheduler/deployment.yaml
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/kosmos-scheduler.yml"
-  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/scheduler/rbac.yaml
-
-  util::wait_for_condition "kosmos scheduler are ready" \
-    "kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system get deploy kosmos-scheduler -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \
-    300
-  echo "cluster $clustername deploy kosmos-scheduler success"
-
-  docker exec ${clustername}-control-plane /bin/sh -c "mv /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes"
+  #  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/virtual-cluster-operator:"${VERSION}"
+  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/node-agent:"${VERSION}"
 }
 
-function load_cluster_images() {
-  local -r clustername=$1
-
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-network-manager:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-controller-manager:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-elector:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-operator:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-agent:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-proxy:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clustertree-cluster-manager:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/scheduler:"${VERSION}"
-}
+function create_node_agent_daemonset() {
+  # insure htpasswd
+  util::cmd_must_exist openssl
+  # generate username and password
+  username=$(openssl rand -hex 5)
+  password=$(openssl rand -base64 12)
+  echo "node-agent生成的用户名: $username"
+  echo "node-agent生成的密码: $password"
+  # Base64 encode the username and password
+  encoded_username=$(echo -n "$username" | base64)
+  encoded_password=$(echo -n "$password" | base64)
+
+  sed -e "s|^  username:.*|  username: ${encoded_username}|g" \
+    -e "s|^  password:.*|  password: ${encoded_password}|g" \
+    -e "w ${ROOT}/environments/node-agent.yaml" "$ROOT"/deploy/node-agent.yaml
 
-function load_kubenetst_cluster_images() {
   local -r clustername=$1
-
-  #  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/virtual-cluster-operator:"${VERSION}"
-  kind load docker-image -n "$clustername" ghcr.io/kosmos-io/node-agent:"${VERSION}"
+  CLUSTER_DIR="${ROOT}/environments/${clustername}"
+  kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/node-agent.yaml"
 }
 
 function delete_cluster() {

hack/install-go.sh

@@ -9,13 +9,13 @@ install_go() {
   echo "Go is not installed. Installing..."
 
   # Specify the Go version you want to install
-  GO_VERSION="1.20"  # Change this to the desired Go version
+  GO_VERSION="1.23.2"  # Change this to the desired Go version
 
   # Set the Go installation path
   GO_INSTALL_PATH="/usr/local"
 
   # Download and install Go
-  curl -O https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz
+  wget https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz
   tar -C $GO_INSTALL_PATH -xzf go$GO_VERSION.linux-amd64.tar.gz
 
   # Set Go environment variables
@@ -35,7 +35,7 @@ if ! command -v go &> /dev/null; then
 fi
 
 # Verify the Go version
-if ! go version | grep -q "go1.20"; then
+if ! go version | grep -q "go1.23.2"; then
   echo "Installed Go version does not match the required version (1.20)."
   install_go
 fi

hack/local-up-kubenest.sh

@@ -31,7 +31,7 @@ done
 KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
 export KUBECONFIG=$KUBECONFIG_PATH/"config"
 
-KIND_IMAGE=${KIND_IMAGE:-"m.daocloud.io/docker.io/kindest/node:v1.27.2"}
+KIND_IMAGE=${KIND_IMAGE:-"kindest/node:v1.27.2"}
 HOST_IPADDRESS=${1:-}
 KUBE_NEST_CLUSTER_NAME="kubenest-cluster"
 CLUSTER_POD_CIDR="10.233.64.0/18"
@@ -40,6 +40,7 @@ CLUSTER_SERVICE_CIDR="10.233.0.0/18"
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 VERSION=${VERSION:-"latest"}
 source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
 source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
 source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
 
@@ -58,8 +59,28 @@ export PATH=$PATH:"${REPO_ROOT}"/_output/bin/"$os"/"$arch"
 # prepare docker image and push to registry
 prepare_docker_image
 
+# create kind cluster
 create_cluster "${KIND_IMAGE}" "${HOST_IPADDRESS}" "${KUBE_NEST_CLUSTER_NAME}" "${CLUSTER_POD_CIDR}" "${CLUSTER_SERVICE_CIDR}" false true
 
+# load images to kind cluster
 load_kubenetst_cluster_images "${KUBE_NEST_CLUSTER_NAME}"
 
+# install sudo command in kind's node container
+# define node name
+node_names=(
+  "${KUBE_NEST_CLUSTER_NAME}-control-plane"
+  "${KUBE_NEST_CLUSTER_NAME}-worker"
+  "${KUBE_NEST_CLUSTER_NAME}-worker2"
+  "${KUBE_NEST_CLUSTER_NAME}-worker3"
+  "${KUBE_NEST_CLUSTER_NAME}-worker4"
+)
+
+# todo execute in parallel
+for node in "${node_names[@]}"
+do
+  echo "Updating and installing sudo on $node..."
+  docker exec -it "$node" bash -c "apt-get update && apt-get install -y sudo"
+done
 
+#step2. create node-agent daemonset in kubernetes
+create_node_agent_daemonset "${KUBE_NEST_CLUSTER_NAME}"