From 80a74d708cb73d4b898c657e2edae04ebfbdf794 Mon Sep 17 00:00:00 2001 From: duanmengkk Date: Wed, 11 Dec 2024 09:14:42 +0800 Subject: [PATCH] improve start script Signed-off-by: duanmengkk --- cluster/images/agent.Dockerfile | 4 +- cluster/images/buildx.agent.Dockerfile | 4 +- deploy/{node-agent.yml => node-agent.yaml} | 48 ++++++- hack/cluster.sh | 149 ++++----------------- hack/install-go.sh | 6 +- hack/local-up-kubenest.sh | 23 +++- 6 files changed, 101 insertions(+), 133 deletions(-) rename deploy/{node-agent.yml => node-agent.yaml} (66%) diff --git a/cluster/images/agent.Dockerfile b/cluster/images/agent.Dockerfile index 9a129f5..221c9f0 100644 --- a/cluster/images/agent.Dockerfile +++ b/cluster/images/agent.Dockerfile @@ -1,4 +1,4 @@ -FROM m.daocloud.io/docker.io/ubuntu AS release-env +FROM docker.io/ubuntu AS release-env ARG BINARY @@ -17,7 +17,7 @@ RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.aliyun.com/ubu sed -i 's|http://archive.ubuntu.com/ubuntu/|http://mirrors.aliyun.com/ubuntu/|' /etc/apt/sources.list RUN apt-get update && \ - apt-get install -y rsync pwgen + apt-get install -y rsync pwgen sudo COPY ${BINARY} /app diff --git a/cluster/images/buildx.agent.Dockerfile b/cluster/images/buildx.agent.Dockerfile index 518a37d..9d7b27b 100644 --- a/cluster/images/buildx.agent.Dockerfile +++ b/cluster/images/buildx.agent.Dockerfile @@ -1,4 +1,4 @@ -FROM m.daocloud.io/docker.io/ubuntu AS release-env +FROM docker.io/ubuntu AS release-env ARG BINARY ARG TARGETPLATFORM @@ -17,7 +17,7 @@ RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.aliyun.com/ubu sed -i 's|http://archive.ubuntu.com/ubuntu/|http://mirrors.aliyun.com/ubuntu/|' /etc/apt/sources.list RUN apt-get update && \ - apt-get install -y rsync pwgen + apt-get install -y rsync pwgen sudo COPY ${TARGETPLATFORM}/${BINARY} /app diff --git a/deploy/node-agent.yml b/deploy/node-agent.yaml similarity index 66% rename from deploy/node-agent.yml rename to deploy/node-agent.yaml index 1a8e608..558eef7 100644 --- a/deploy/node-agent.yml +++ b/deploy/node-agent.yaml @@ -1,3 +1,38 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kosmos-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubenest-node-agent + namespace: kosmos-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubenest-node-agent +rules: + - apiGroups: ['*'] + resources: ['*'] + verbs: ["*"] + - nonResourceURLs: ['*'] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubenest-node-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubenest-node-agent +subjects: + - kind: ServiceAccount + name: kubenest-node-agent + namespace: kosmos-system +--- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -15,6 +50,7 @@ spec: hostPID: true # access host pid hostIPC: true # access host ipc hostNetwork: true # access host network + serviceAccountName: kubenest-node-agent tolerations: - operator: Exists # run on all nodes initContainers: @@ -70,4 +106,14 @@ spec: - name: systemd-path hostPath: path: /etc/systemd/system - type: DirectoryOrCreate \ No newline at end of file + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: Secret +metadata: + name: node-agent-secret + namespace: kosmos-system +type: kubernetes.io/basic-auth +data: + username: {{ .USERNAME }} + password: {{ .PASSWORD }} diff --git a/hack/cluster.sh b/hack/cluster.sh index 53835ef..975be63 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -13,7 +13,7 @@ REUSE=${REUSE:-false} USE_LOCAL_ARTIFACTS=${USE_LOCAL_ARTIFACTS:-true} VERSION=${VERSION:-latest} -CN_ZONE=${CN_ZONE:-true} +CN_ZONE=${CN_ZONE:-false} source "$(dirname "${BASH_SOURCE[0]}")/util.sh" # default cert and key for node server https @@ -122,18 +122,18 @@ function prepare_docker_image() { if [ "${CN_ZONE}" == false ]; then # 使用 Calico 的官方镜像源 - local calico_prefix="calico" + local calico_prefix="" local operator_prefix="quay.io" else # 使用 DaoCloud 镜像源 - calico_prefix="docker.m.daocloud.io" + calico_prefix="docker.m.daocloud.io/" operator_prefix="quay.m.daocloud.io" fi # 拉取和标记 Calico 镜像 for image in "${calico_images[@]}"; do - docker pull "${calico_prefix}/${image}:${version}" - docker tag "${calico_prefix}/${image}:${version}" "${image}:${version}" + docker pull "${calico_prefix}${image}:${version}" + docker tag "${calico_prefix}${image}:${version}" "${image}:${version}" done # 拉取和标记 Operator 镜像 @@ -220,131 +220,32 @@ function create_cluster() { echo "all node ready" } -function join_cluster() { - local host_cluster=$1 - local member_cluster=$2 - local kubeconfig_path="${ROOT}/environments/${member_cluster}/kubeconfig" - local hostConfig_path="${ROOT}/environments/${host_cluster}/kubeconfig" - local base64_kubeconfig=$(util::get_base64_kubeconfig <"$kubeconfig_path") - echo " base64 kubeconfig successfully converted: $base64_kubeconfig " - - local common_metadata="" - if [ "$host_cluster" == "$member_cluster" ]; then - common_metadata="annotations: - kosmos.io/cluster-role: root" - fi - - cat < 0) exit 0; else exit 1}'" \ - 300 -} - -function deploy_cluster() { +function load_kubenetst_cluster_images() { local -r clustername=$1 - CLUSTER_DIR="${ROOT}/environments/${clustername}" - - load_cluster_images "$clustername" - - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/clusterlink-namespace.yml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/kosmos-rbac.yml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/crds - util::wait_for_crd clusternodes.kosmos.io clusters.kosmos.io clusterdistributionpolicies.kosmos.io distributionpolicies.kosmos.io - - sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clusterlink-network-manager.yml" "$ROOT"/deploy/clusterlink-network-manager.yml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clusterlink-network-manager.yml" - - echo "cluster $clustername deploy clusterlink success" - sed -e "s|__VERSION__|$VERSION|g" -e "s|__CERT__|$CERT|g" -e "s|__KEY__|$KEY|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clustertree-cluster-manager.yml" - - echo "cluster $clustername deploy clustertree success" - - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system delete secret controlpanel-config || true - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system create secret generic controlpanel-config --from-file=kubeconfig="${ROOT}/environments/cluster-host/kubeconfig" - sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clusterlink-operator.yml" "$ROOT"/deploy/clusterlink-operator.yml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/clusterlink-operator.yml" - - echo "cluster $clustername deploy clusterlink-operator success" - - sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/kosmos-scheduler.yml" "$ROOT"/deploy/scheduler/deployment.yaml - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/kosmos-scheduler.yml" - kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "$ROOT"/deploy/scheduler/rbac.yaml - - util::wait_for_condition "kosmos scheduler are ready" \ - "kubectl --kubeconfig $CLUSTER_DIR/kubeconfig -n kosmos-system get deploy kosmos-scheduler -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \ - 300 - echo "cluster $clustername deploy kosmos-scheduler success" - - docker exec ${clustername}-control-plane /bin/sh -c "mv /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes" + # kind load docker-image -n "$clustername" ghcr.io/kosmos-io/virtual-cluster-operator:"${VERSION}" + kind load docker-image -n "$clustername" ghcr.io/kosmos-io/node-agent:"${VERSION}" } -function load_cluster_images() { - local -r clustername=$1 - - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-network-manager:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-controller-manager:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-elector:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-operator:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-agent:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-proxy:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clustertree-cluster-manager:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/scheduler:"${VERSION}" -} +function create_node_agent_daemonset() { + # insure htpasswd + util::cmd_must_exist openssl + # generate username and password + username=$(openssl rand -hex 5) + password=$(openssl rand -base64 12) + echo "node-agent生成的用户名: $username" + echo "node-agent生成的密码: $password" + # Base64 encode the username and password + encoded_username=$(echo -n "$username" | base64) + encoded_password=$(echo -n "$password" | base64) + + sed -e "s|^ username:.*| username: ${encoded_username}|g" \ + -e "s|^ password:.*| password: ${encoded_password}|g" \ + -e "w ${ROOT}/environments/node-agent.yaml" "$ROOT"/deploy/node-agent.yaml -function load_kubenetst_cluster_images() { local -r clustername=$1 - - # kind load docker-image -n "$clustername" ghcr.io/kosmos-io/virtual-cluster-operator:"${VERSION}" - kind load docker-image -n "$clustername" ghcr.io/kosmos-io/node-agent:"${VERSION}" + CLUSTER_DIR="${ROOT}/environments/${clustername}" + kubectl --kubeconfig $CLUSTER_DIR/kubeconfig apply -f "${ROOT}/environments/node-agent.yaml" } function delete_cluster() { diff --git a/hack/install-go.sh b/hack/install-go.sh index bc5c4d7..875e14f 100644 --- a/hack/install-go.sh +++ b/hack/install-go.sh @@ -9,13 +9,13 @@ install_go() { echo "Go is not installed. Installing..." # Specify the Go version you want to install - GO_VERSION="1.20" # Change this to the desired Go version + GO_VERSION="1.23.2" # Change this to the desired Go version # Set the Go installation path GO_INSTALL_PATH="/usr/local" # Download and install Go - curl -O https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz + wget https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz tar -C $GO_INSTALL_PATH -xzf go$GO_VERSION.linux-amd64.tar.gz # Set Go environment variables @@ -35,7 +35,7 @@ if ! command -v go &> /dev/null; then fi # Verify the Go version -if ! go version | grep -q "go1.20"; then +if ! go version | grep -q "go1.23.2"; then echo "Installed Go version does not match the required version (1.20)." install_go fi diff --git a/hack/local-up-kubenest.sh b/hack/local-up-kubenest.sh index e874b33..f6172b6 100755 --- a/hack/local-up-kubenest.sh +++ b/hack/local-up-kubenest.sh @@ -31,7 +31,7 @@ done KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} export KUBECONFIG=$KUBECONFIG_PATH/"config" -KIND_IMAGE=${KIND_IMAGE:-"m.daocloud.io/docker.io/kindest/node:v1.27.2"} +KIND_IMAGE=${KIND_IMAGE:-"kindest/node:v1.27.2"} HOST_IPADDRESS=${1:-} KUBE_NEST_CLUSTER_NAME="kubenest-cluster" CLUSTER_POD_CIDR="10.233.64.0/18" @@ -40,6 +40,7 @@ CLUSTER_SERVICE_CIDR="10.233.0.0/18" REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. VERSION=${VERSION:-"latest"} source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh" +source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh" source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh" source "$(dirname "${BASH_SOURCE[0]}")/util.sh" @@ -58,8 +59,28 @@ export PATH=$PATH:"${REPO_ROOT}"/_output/bin/"$os"/"$arch" # prepare docker image and push to registry prepare_docker_image +# create kind cluster create_cluster "${KIND_IMAGE}" "${HOST_IPADDRESS}" "${KUBE_NEST_CLUSTER_NAME}" "${CLUSTER_POD_CIDR}" "${CLUSTER_SERVICE_CIDR}" false true +# load images to kind cluster load_kubenetst_cluster_images "${KUBE_NEST_CLUSTER_NAME}" +# install sudo command in kind's node container +# define node name +node_names=( + "${KUBE_NEST_CLUSTER_NAME}-control-plane" + "${KUBE_NEST_CLUSTER_NAME}-worker" + "${KUBE_NEST_CLUSTER_NAME}-worker2" + "${KUBE_NEST_CLUSTER_NAME}-worker3" + "${KUBE_NEST_CLUSTER_NAME}-worker4" +) + +# todo execute in parallel +for node in "${node_names[@]}" +do + echo "Updating and installing sudo on $node..." + docker exec -it "$node" bash -c "apt-get update && apt-get install -y sudo" +done +#step2. create node-agent daemonset in kubernetes +create_node_agent_daemonset "${KUBE_NEST_CLUSTER_NAME}"