diff --git a/applications/calico/example/Clusterfile-bare.yaml b/applications/calico/example/Clusterfile-bare.yaml deleted file mode 100644 index 0400dee57da..00000000000 --- a/applications/calico/example/Clusterfile-bare.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright © 2021 Alibaba Group Holding Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: sealer.aliyun.com/v1alpha1 -kind: Cluster -metadata: - name: my-cluster -spec: - image: kubernetes-withcalico:v1.19.9 - masters: - ipList: - - x.x.x.x - network: - podCIDR: 100.64.0.0/10 - svcCIDR: 10.96.0.0/22 - nodes: - ipList: - - x.x.x.x - ssh: - passwd: xxx - provider: BAREMETAL ---- -apiVersion: sealer.aliyun.com/v1alpha1 -kind: Config -metadata: - name: calico -spec: - path: etc/custom-resources.yaml - data: | - apiVersion: operator.tigera.io/v1 - kind: Installation - metadata: - name: default - spec: - # Configures Calico networking. - calicoNetwork: - # Note: The ipPools section cannot be modified post-install. - ipPools: - - blockSize: 26 - cidr: 100.64.0.0/10 - encapsulation: IPIP - natOutgoing: Enabled - nodeSelector: all() - nodeAddressAutodetectionV4: - interface: "eth.*|en.*" \ No newline at end of file diff --git a/applications/calico/example/Clusterfile.yaml b/applications/calico/example/Clusterfile.yaml index 9fd210e098c..7c104b000f0 100644 --- a/applications/calico/example/Clusterfile.yaml +++ b/applications/calico/example/Clusterfile.yaml @@ -12,30 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: sealer.aliyun.com/v1alpha1 +apiVersion: sealer.cloud/v2 kind: Cluster metadata: - name: my-cluster + name: default-kubernetes-cluster spec: - image: kubernetes-withcalico:v1.19.9 - provider: ALI_CLOUD - masters: - count: "1" - cpu: "4" - dataDisks: - - "100" - memory: "4" - systemDisk: "100" - network: - podCIDR: 100.64.0.0/10 - svcCIDR: 10.96.0.0/22 - nodes: - count: "1" - cpu: "4" - dataDisks: - - "100" - memory: "4" - systemDisk: "100" + image: kubernetes:v1.19.8 + ssh: + passwd: xxx + port: "2222" + hosts: + - ips: [ 192.168.0.2 ] + roles: [ master ] + ssh: + passwd: yyy + port: "22" + - ips: [ 192.168.0.3,192.168.0.4 ] + roles: [ master ] + - ips: [ 192.168.0.5 ] + roles: [ node ] --- apiVersion: sealer.aliyun.com/v1alpha1 kind: Config @@ -49,14 +44,11 @@ spec: metadata: name: default spec: - # Configures Calico networking. + registry: sea.hub:5000 calicoNetwork: - # Note: The ipPools section cannot be modified post-install. ipPools: - - blockSize: 26 - cidr: 100.64.0.0/10 - encapsulation: IPIP - natOutgoing: Enabled - nodeSelector: all() - nodeAddressAutodetectionV4: - interface: "eth.*|en.*" + - blockSize: 26 + cidr: 100.64.0.0/10 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() diff --git a/pkg/filesystem/rootfs/containerd/etc/dump-config.toml b/pkg/filesystem/rootfs/containerd/etc/dump-config.toml new file mode 100644 index 00000000000..384994cf2e7 --- /dev/null +++ b/pkg/filesystem/rootfs/containerd/etc/dump-config.toml @@ -0,0 +1,10 @@ +version = 2 +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "sea.hub:5000/pause:3.6" + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/docker/certs.d/" + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."sea.hub:5000".tls] + insecure_skip_verify = true + ca_file = "/etc/docker/certs.d/sea.hub:5000/sea.hub.crt" \ No newline at end of file diff --git a/pkg/filesystem/rootfs/containerd/scripts/clean.sh b/pkg/filesystem/rootfs/containerd/scripts/clean.sh new file mode 100644 index 00000000000..8b3a91f8fac --- /dev/null +++ b/pkg/filesystem/rootfs/containerd/scripts/clean.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright © 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +systemctl stop containerd + +rm -f /usr/bin/conntrack +rm -f /usr/bin/kubelet-pre-start.sh +rm -f /usr/bin/containerd +rm -rf /etc/containerd +rm -f /usr/bin/containerd-shim +rm -f /usr/bin/containerd-shim-runc-v2 +rm -f /usr/bin/crictl +rm -f /usr/bin/ctr + +rm -f /usr/bin/kubeadm +rm -f /usr/bin/kubetcl +rm -f /usr/bin/kubelet +rm -f /usr/bin/rootlesskit +rm -f /usr/bin/rootlesskit-docker-proxy +rm -f /usr/bin/runc +rm -f /usr/bin/vpnkit +rm -f /usr/bin/containerd-rootless-setuptool.sh +rm -f /usr/bin/containerd-rootless.sh +rm -f /usr/bin/nerdctl + +rm -f /etc/sysctl.d/k8s.conf +rm -f /etc/systemd/system/kubelet.service +rm -rf /etc/systemd/system/kubelet.service.d +rm -rf /var/lib/kubelet/ +rm -f /var/lib/kubelet/config.yaml +rm -f /etc/crictl.yaml diff --git a/pkg/filesystem/rootfs/containerd/scripts/containerd.sh b/pkg/filesystem/rootfs/containerd/scripts/containerd.sh new file mode 100644 index 00000000000..4a7b3b24129 --- /dev/null +++ b/pkg/filesystem/rootfs/containerd/scripts/containerd.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright © 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +if ! [ -x /usr/local/bin/ctr ]; then + tar -xvzf ../cri/containerd.tar.gz -C / + [ -f /usr/lib64/libseccomp.so.2 ] || cp -rf ../lib64/lib* /usr/lib64/ + systemctl enable containerd.service + systemctl restart containerd.service +fi + +mkdir -p /etc/containerd + +sed -i "s/sea.hub/${1:-sea.hub}/g" ../etc/dump-config.toml +sed -i "s/5000/${2:-5000}/g" ../etc/dump-config.toml + +#add cri sandbox image and sea.hub registry cert path +##sandbox_image = "sea.hub:5000/pause:3.6" custom setup +containerd --config ../etc/dump-config.toml config dump > /etc/containerd/config.toml + +systemctl restart containerd.service \ No newline at end of file diff --git a/pkg/filesystem/rootfs/containerd/scripts/init-registry.sh b/pkg/filesystem/rootfs/containerd/scripts/init-registry.sh new file mode 100644 index 00000000000..3f40971f8ee --- /dev/null +++ b/pkg/filesystem/rootfs/containerd/scripts/init-registry.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright © 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e +set -x +# prepare registry storage as directory +cd $(dirname "$0") + +REGISTRY_PORT=${1-5000} +VOLUME=${2-/var/lib/registry} +REGISTRY_DOMAIN=${3-sea.hub} + +container=sealer-registry +rootfs=$(dirname "$(pwd)") +config="$rootfs/etc/registry_config.yml" +htpasswd="$rootfs/etc/registry_htpasswd" +certs_dir="$rootfs/certs" +image_dir="$rootfs/images" + +mkdir -p "$VOLUME" || true + +startRegistry() { + n=1 + while (( n <= 3 )) + do + echo "attempt to start registry" + (nerdctl start $container && break) || (( n < 3)) + (( n++ )) + sleep 3 + done +} + +load_images() { +for image in "$image_dir"/* +do + if [ -f "${image}" ] + then + nerdctl load -i "${image}" + fi +done +} + +check_registry() { + n=1 + while (( n <= 3 )) + do + (nerdctl inspect sealer-registry | grep "\"Status\": \"running\"") && break + if [[ $n -eq 3 ]]; then + echo "sealer-registry is not running, status: $registry_status" + exit 1 + fi + (( n++ )) + sleep 3 + done +} + +load_images + +## rm container if exist. +nerdctl rm -f $container || true +## +rm -rf /var/lib/nerdctl/1935db59/names/default/$container + +regArgs="-d --restart=always \ +--net=host \ +--name $container \ +-v $certs_dir:/certs \ +-v $VOLUME:/var/lib/registry \ +-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_DOMAIN.crt \ +-e REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_DOMAIN.key" + +if [ -f $config ]; then + sed -i "s/5000/$1/g" $config + regArgs="$regArgs \ + -v $config:/etc/docker/registry/config.yml" +fi + +if [ -f $htpasswd ]; then + nerdctl run $regArgs \ + -v $htpasswd:/htpasswd \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" registry:2.7.1 || startRegistry +else + nerdctl run $regArgs registry:2.7.1 || startRegistry +fi + +check_registry \ No newline at end of file diff --git a/pkg/filesystem/rootfs/containerd/scripts/init.sh b/pkg/filesystem/rootfs/containerd/scripts/init.sh new file mode 100644 index 00000000000..5916bbf36e3 --- /dev/null +++ b/pkg/filesystem/rootfs/containerd/scripts/init.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright © 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +#STORAGE=${1:-/var/lib/docker} compatible docker +REGISTRY_DOMAIN=${2-sea.hub} +REGISTRY_PORT=${3-5000} + +# Install containerd +chmod a+x containerd.sh +sh containerd.sh "$REGISTRY_DOMAIN" "$REGISTRY_PORT" + +# Modify kubelet conf +mkdir -p /etc/systemd/system/kubelet.service.d + +if grep "SystemdCgroup = true" /etc/containerd/config.toml &>/dev/null; then + driver=systemd +else + driver=cgroupfs +fi + +cat >/etc/systemd/system/kubelet.service.d/containerd.conf <