support contained cloud image (#1336)

applications/calico/example/Clusterfile.yaml

@@ -12,30 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: sealer.aliyun.com/v1alpha1
+apiVersion: sealer.cloud/v2
 kind: Cluster
 metadata:
-  name: my-cluster
+  name: default-kubernetes-cluster
 spec:
-  image: kubernetes-withcalico:v1.19.9
-  provider: ALI_CLOUD
-  masters:
-    count: "1"
-    cpu: "4"
-    dataDisks:
-    - "100"
-    memory: "4"
-    systemDisk: "100"
-  network:
-    podCIDR: 100.64.0.0/10
-    svcCIDR: 10.96.0.0/22
-  nodes:
-    count: "1"
-    cpu: "4"
-    dataDisks:
-    - "100"
-    memory: "4"
-    systemDisk: "100"    
+  image: kubernetes:v1.19.8
+  ssh:
+    passwd: xxx
+    port: "2222"
+  hosts:
+    - ips: [ 192.168.0.2 ]
+      roles: [ master ]
+      ssh:
+        passwd: yyy
+        port: "22"
+    - ips: [ 192.168.0.3,192.168.0.4 ]
+      roles: [ master ]
+    - ips: [ 192.168.0.5 ]
+      roles: [ node ]
 ---
 apiVersion: sealer.aliyun.com/v1alpha1
 kind: Config
@@ -49,14 +44,11 @@ spec:
     metadata:
       name: default
     spec:
-      # Configures Calico networking.
+      registry: sea.hub:5000
       calicoNetwork:
-        # Note: The ipPools section cannot be modified post-install.
         ipPools:
-        - blockSize: 26
-          cidr: 100.64.0.0/10
-          encapsulation: IPIP
-          natOutgoing: Enabled
-          nodeSelector: all()
-        nodeAddressAutodetectionV4:
-          interface: "eth.*|en.*"
+          - blockSize: 26
+            cidr: 100.64.0.0/10
+            encapsulation: VXLANCrossSubnet
+            natOutgoing: Enabled
+            nodeSelector: all()

pkg/filesystem/rootfs/containerd/etc/dump-config.toml

@@ -0,0 +1,10 @@
+version = 2
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "sea.hub:5000/pause:3.6"
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/docker/certs.d/"
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."sea.hub:5000".tls]
+          insecure_skip_verify = true
+          ca_file = "/etc/docker/certs.d/sea.hub:5000/sea.hub.crt"

pkg/filesystem/rootfs/containerd/scripts/clean.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright © 2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+systemctl stop containerd
+
+rm -f /usr/bin/conntrack
+rm -f /usr/bin/kubelet-pre-start.sh
+rm -f /usr/bin/containerd
+rm -rf /etc/containerd
+rm -f /usr/bin/containerd-shim
+rm -f /usr/bin/containerd-shim-runc-v2
+rm -f /usr/bin/crictl
+rm -f /usr/bin/ctr
+
+rm -f /usr/bin/kubeadm
+rm -f /usr/bin/kubetcl
+rm -f /usr/bin/kubelet
+rm -f /usr/bin/rootlesskit
+rm -f /usr/bin/rootlesskit-docker-proxy
+rm -f /usr/bin/runc
+rm -f /usr/bin/vpnkit
+rm -f /usr/bin/containerd-rootless-setuptool.sh
+rm -f /usr/bin/containerd-rootless.sh
+rm -f /usr/bin/nerdctl
+
+rm -f /etc/sysctl.d/k8s.conf
+rm -f /etc/systemd/system/kubelet.service
+rm -rf /etc/systemd/system/kubelet.service.d
+rm -rf /var/lib/kubelet/
+rm -f /var/lib/kubelet/config.yaml
+rm -f /etc/crictl.yaml

pkg/filesystem/rootfs/containerd/scripts/containerd.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright © 2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+if ! [ -x /usr/local/bin/ctr ]; then
+  tar  -xvzf ../cri/containerd.tar.gz -C /
+  [ -f /usr/lib64/libseccomp.so.2 ] || cp -rf ../lib64/lib* /usr/lib64/
+  systemctl enable  containerd.service
+  systemctl restart containerd.service
+fi
+
+mkdir -p /etc/containerd
+
+sed -i "s/sea.hub/${1:-sea.hub}/g" ../etc/dump-config.toml
+sed -i "s/5000/${2:-5000}/g" ../etc/dump-config.toml
+
+#add cri sandbox image and sea.hub registry cert path
+##sandbox_image = "sea.hub:5000/pause:3.6" custom setup
+containerd --config ../etc/dump-config.toml config dump > /etc/containerd/config.toml
+
+systemctl restart containerd.service

pkg/filesystem/rootfs/containerd/scripts/init-registry.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+# Copyright © 2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -e
+set -x
+# prepare registry storage as directory
+cd $(dirname "$0")
+
+REGISTRY_PORT=${1-5000}
+VOLUME=${2-/var/lib/registry}
+REGISTRY_DOMAIN=${3-sea.hub}
+
+container=sealer-registry
+rootfs=$(dirname "$(pwd)")
+config="$rootfs/etc/registry_config.yml"
+htpasswd="$rootfs/etc/registry_htpasswd"
+certs_dir="$rootfs/certs"
+image_dir="$rootfs/images"
+
+mkdir -p "$VOLUME" || true
+
+startRegistry() {
+    n=1
+    while (( n <= 3 ))
+    do
+        echo "attempt to start registry"
+        (nerdctl start $container && break) || (( n < 3))
+        (( n++ ))
+        sleep 3
+    done
+}
+
+load_images() {
+for image in "$image_dir"/*
+do
+ if [ -f "${image}" ]
+ then
+  nerdctl load -i "${image}"
+ fi
+done
+}
+
+check_registry() {
+    n=1
+    while (( n <= 3 ))
+    do
+        (nerdctl inspect sealer-registry | grep "\"Status\": \"running\"") && break
+        if [[ $n -eq 3 ]]; then
+           echo "sealer-registry is not running, status: $registry_status"
+           exit 1
+        fi
+        (( n++ ))
+        sleep 3
+    done
+}
+
+load_images
+
+## rm container if exist.
+nerdctl rm -f $container || true
+##
+rm -rf /var/lib/nerdctl/1935db59/names/default/$container
+
+regArgs="-d --restart=always \
+--net=host \
+--name $container \
+-v $certs_dir:/certs \
+-v $VOLUME:/var/lib/registry \
+-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_DOMAIN.crt \
+-e REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_DOMAIN.key"
+
+if [ -f $config ]; then
+    sed -i "s/5000/$1/g" $config
+    regArgs="$regArgs \
+    -v $config:/etc/docker/registry/config.yml"
+fi
+
+if [ -f $htpasswd ]; then
+    nerdctl run $regArgs \
+            -v $htpasswd:/htpasswd \
+            -e REGISTRY_AUTH=htpasswd \
+            -e REGISTRY_AUTH_HTPASSWD_PATH=/htpasswd \
+            -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" registry:2.7.1 || startRegistry
+else
+    nerdctl run $regArgs registry:2.7.1 || startRegistry
+fi
+
+check_registry

pkg/filesystem/rootfs/containerd/scripts/init.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright © 2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+#STORAGE=${1:-/var/lib/docker} compatible docker
+REGISTRY_DOMAIN=${2-sea.hub}
+REGISTRY_PORT=${3-5000}
+
+# Install containerd
+chmod a+x containerd.sh
+sh containerd.sh "$REGISTRY_DOMAIN" "$REGISTRY_PORT"
+
+# Modify kubelet conf
+mkdir -p /etc/systemd/system/kubelet.service.d
+
+if grep "SystemdCgroup = true" /etc/containerd/config.toml &>/dev/null; then
+  driver=systemd
+else
+  driver=cgroupfs
+fi
+
+cat >/etc/systemd/system/kubelet.service.d/containerd.conf <<eof
+[Service]
+Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --cgroup-driver=${driver} --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --image-service-endpoint=unix:///run/containerd/containerd.sock"
+eof
+
+chmod a+x init-kube.sh
+sh init-kube.sh

pkg/filesystem/rootfs/docker/scripts/clean.sh

@@ -30,6 +30,7 @@ rm -f /usr/bin/docker
 rm -f /usr/bin/docker-init
 rm -f /usr/bin/docker-proxy
 rm -f /usr/bin/dockerd
+
 rm -f /usr/bin/kubeadm
 rm -f /usr/bin/kubetcl
 rm -f /usr/bin/kubelet

pkg/filesystem/rootfs/docker/scripts/init.sh

@@ -14,16 +14,17 @@
 # limitations under the License.
 
 
+set -e
+set -x
 
 STORAGE=${1:-/var/lib/docker}
 REGISTRY_DOMAIN=${2-sea.hub}
 REGISTRY_PORT=${3-5000}
 
-
 # Install docker
 chmod a+x docker.sh
 #./docker.sh  /var/docker/lib  sealer.hub 5001
-bash docker.sh ${STORAGE} ${REGISTRY_DOMAIN} $REGISTRY_PORT
+bash docker.sh "$STORAGE" "$REGISTRY_DOMAIN" "$REGISTRY_PORT"
 
 chmod a+x init-kube.sh
 

pkg/filesystem/rootfs/rootfs/scripts/init-kube.sh

@@ -91,4 +91,4 @@ mkdir -p /etc/systemd/system
 cp ../etc/kubelet.service /etc/systemd/system/
 [ -d /etc/systemd/system/kubelet.service.d ] || mkdir /etc/systemd/system/kubelet.service.d
 cp ../etc/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/
-systemctl enable kubelet
+systemctl enable kubelet