From 193bba5277e1b412677bb2ae58af0ea7d55b8c14 Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Tue, 24 Aug 2021 15:41:32 +0800 Subject: [PATCH] Add containerd config files (#170) * Add containerd config files * Add containerd.service file * Bump go version from 1.13 to 1.16 * Ignore G306 for go sec --- .github/workflows/pull-request.yaml | 10 +- .github/workflows/release.yaml | 2 +- go.mod | 6 +- go.sum | 9 +- kubectl-plugin/install/containerd/config.toml | 11 ++ .../install/containerd/containerd.service | 22 +++ kubectl-plugin/install/containerd/crictl.yaml | 6 + .../install/containerd/kk_config.yaml | 31 ++++ kubectl-plugin/install/containerd/resource.go | 38 +++++ .../install/containerd/resource_test.go | 13 ++ kubectl-plugin/install/kind.go | 7 + kubectl-plugin/install/kk.go | 138 ++++++++++++++++-- kubectl-plugin/install/kk_test.go | 22 ++- 13 files changed, 285 insertions(+), 30 deletions(-) create mode 100644 kubectl-plugin/install/containerd/config.toml create mode 100644 kubectl-plugin/install/containerd/containerd.service create mode 100644 kubectl-plugin/install/containerd/crictl.yaml create mode 100644 kubectl-plugin/install/containerd/kk_config.yaml create mode 100644 kubectl-plugin/install/containerd/resource.go create mode 100644 kubectl-plugin/install/containerd/resource_test.go diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 4ae5f25..c2b776e 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -10,10 +10,10 @@ jobs: name: Build runs-on: macos-10.15 steps: - - name: Set up Go 1.13 + - name: Set up Go 1.16 uses: actions/setup-go@v2.1.3 with: - go-version: 1.13 + go-version: 1.16 id: go - name: Check out code into the Go module directory uses: actions/checkout@v2.3.4 @@ -29,10 +29,10 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - name: Set up Go 1.13 + - name: Set up Go 1.16 uses: actions/setup-go@v2.1.3 with: - go-version: 1.13 + go-version: 1.16 id: go - name: Check out code into the Go module directory uses: actions/checkout@v2.3.4 @@ -51,7 +51,7 @@ jobs: - name: Run Gosec Security Scanner uses: securego/gosec@master with: - args: '-exclude=G402,G204,G304,G110 ./...' + args: '-exclude=G402,G204,G304,G110,G306 ./...' CodeQL: name: CodeQL runs-on: ubuntu-latest diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fbc83de..d9eca63 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2.1.3 with: - go-version: 1.13.x + go-version: 1.16.x - name: Upgrade upx run: | # try to fix https://github.com/jenkins-zh/jenkins-cli/issues/493 diff --git a/go.mod b/go.mod index f8dde45..aa710b8 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/linuxsuren/ks -go 1.15 +go 1.16 require ( - github.com/AlecAivazis/survey/v2 v2.2.16 + github.com/AlecAivazis/survey/v2 v2.3.1 github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible @@ -11,7 +11,7 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/linuxsuren/cobra-extension v0.0.10 github.com/linuxsuren/go-cli-alias v0.0.6 - github.com/linuxsuren/http-downloader v0.0.32-0.20210822064615-b730515cf035 + github.com/linuxsuren/http-downloader v0.0.33 github.com/mitchellh/copystructure v1.1.1 // indirect github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index b500f1f..2ce6b64 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AlecAivazis/survey/v2 v2.2.2/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= -github.com/AlecAivazis/survey/v2 v2.2.16 h1:KJ4fLFqY/NfR5OaFLcf4pThxrlV2YCHGCnCHAKLsJ+U= -github.com/AlecAivazis/survey/v2 v2.2.16/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg= +github.com/AlecAivazis/survey/v2 v2.3.1 h1:lzkuHA60pER7L4eYL8qQJor4bUWlJe4V0gqAT19tdOA= +github.com/AlecAivazis/survey/v2 v2.3.1/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -143,7 +143,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -367,8 +366,8 @@ github.com/linuxsuren/go-cli-alias v0.0.6 h1:PGYf1xqy5GzGINeRhiZR1PTlp09cbyC66MG github.com/linuxsuren/go-cli-alias v0.0.6/go.mod h1:Sa7xNUI72BgHTcywDU8MXVSH1q72SLdMALZSSANwuUM= github.com/linuxsuren/http-downloader v0.0.2-0.20201207132639-19888a6beaec/go.mod h1:zRZY9FCDBuYNDxbI2Ny5suasZsMk7J6q9ecQ3V3PIqI= github.com/linuxsuren/http-downloader v0.0.6/go.mod h1:xxgh2OE7WGL9TwDE9L8Gh7Lqq9fFPuHbh5tofUitEfE= -github.com/linuxsuren/http-downloader v0.0.32-0.20210822064615-b730515cf035 h1:pjzJ/FZtQ+DpW2p8gEX/GnamSELEJlR484C1QkfeXUw= -github.com/linuxsuren/http-downloader v0.0.32-0.20210822064615-b730515cf035/go.mod h1:bGdGvSNCmk7LRuhffIK/ZsAIGpKmrcgJqEjdM8lJzdc= +github.com/linuxsuren/http-downloader v0.0.33 h1:ivAVsVS5zEr8F/oPCrdRdtFP+DLWQjGmVA9AVnUu/vo= +github.com/linuxsuren/http-downloader v0.0.33/go.mod h1:qgMo/GRgQlU6EnIQ+Ul6e0rEJiDFM6+Jcz63ZH2Dhw4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= diff --git a/kubectl-plugin/install/containerd/config.toml b/kubectl-plugin/install/containerd/config.toml new file mode 100644 index 0000000..be03a28 --- /dev/null +++ b/kubectl-plugin/install/containerd/config.toml @@ -0,0 +1,11 @@ +# /etc/containerd/config.toml +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "kubesphere/pause:3.2" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" diff --git a/kubectl-plugin/install/containerd/containerd.service b/kubectl-plugin/install/containerd/containerd.service new file mode 100644 index 0000000..c059e97 --- /dev/null +++ b/kubectl-plugin/install/containerd/containerd.service @@ -0,0 +1,22 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target + +[Service] +ExecStartPre=/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd +Restart=always +RestartSec=5 +Delegate=yes +KillMode=process +OOMScoreAdjust=-999 +LimitNOFILE=1048576 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity + +[Install] +WantedBy=multi-user.target diff --git a/kubectl-plugin/install/containerd/crictl.yaml b/kubectl-plugin/install/containerd/crictl.yaml new file mode 100644 index 0000000..cb748aa --- /dev/null +++ b/kubectl-plugin/install/containerd/crictl.yaml @@ -0,0 +1,6 @@ +# /etc/crictl.yaml +runtime-endpoint: unix:///run/containerd/containerd.sock +image-endpoint: unix:///run/containerd/containerd.sock +timeout: 2 +debug: false +pull-image-on-create: false diff --git a/kubectl-plugin/install/containerd/kk_config.yaml b/kubectl-plugin/install/containerd/kk_config.yaml new file mode 100644 index 0000000..87d8dd7 --- /dev/null +++ b/kubectl-plugin/install/containerd/kk_config.yaml @@ -0,0 +1,31 @@ +apiVersion: kubekey.kubesphere.io/v1alpha1 +kind: Cluster +metadata: + name: kk-config-file +spec: + hosts: + - {name: node1, address: {{.address}}, inetnalAddress: {{.address}}} + roleGroups: + etcd: + - node1 + master: + - node1 + worker: + - node1 + controlPlaneEndpoint: + domain: lb.kubesphere.local + address: "" + port: 6443 + kubernetes: + version: v1.19.8 + imageRepo: kubesphere + clusterName: cluster.local + containerManager: {{.container}} + network: + plugin: calico + kubePodsCIDR: 10.233.64.0/18 + kubeServiceCIDR: 10.233.0.0/18 + registry: + registryMirrors: [] + insecureRegistries: [] + addons: [] diff --git a/kubectl-plugin/install/containerd/resource.go b/kubectl-plugin/install/containerd/resource.go new file mode 100644 index 0000000..6bd37a2 --- /dev/null +++ b/kubectl-plugin/install/containerd/resource.go @@ -0,0 +1,38 @@ +package containerd + +import ( + // Enable go embed + _ "embed" +) + +//go:embed config.toml +var configToml string + +// GetConfigToml returns the default containerd config file content +func GetConfigToml() string { + return configToml +} + +//go:embed crictl.yaml +var crictl string + +// GetCrictl returns the default crictl config file content +func GetCrictl() string { + return crictl +} + +//go:embed kk_config.yaml +var kkConfig string + +// GetKKConfig returns the default kubekey config file content +func GetKKConfig() string { + return kkConfig +} + +//go:embed containerd.service +var containerdService string + +// GetContainerdService returns the default containerd.service file content +func GetContainerdService() string { + return containerdService +} diff --git a/kubectl-plugin/install/containerd/resource_test.go b/kubectl-plugin/install/containerd/resource_test.go new file mode 100644 index 0000000..6408840 --- /dev/null +++ b/kubectl-plugin/install/containerd/resource_test.go @@ -0,0 +1,13 @@ +package containerd + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestEmebdContent(t *testing.T) { + assert.NotEmpty(t, GetCrictl(), "crictl config file is empty") + assert.NotEmpty(t, GetConfigToml(), "containerd config file is empty") + assert.NotEmpty(t, GetKKConfig(), "kk config file is empty") + assert.NotEmpty(t, GetContainerdService(), "containerd.service is empty") +} diff --git a/kubectl-plugin/install/kind.go b/kubectl-plugin/install/kind.go index 587c279..fa34522 100644 --- a/kubectl-plugin/install/kind.go +++ b/kubectl-plugin/install/kind.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "html/template" "os" + "runtime" "sync" ) @@ -35,6 +36,8 @@ ks install kind --nightly latest --components DevOps`, flags.BoolVarP(&opt.Reset, "reset", "", false, "") flags.StringVarP(&opt.Nightly, "nightly", "", "", "Supported date format is '20200101', or you can use 'latest' which means yesterday") + flags.BoolVarP(&opt.fetch, "fetch", "", true, + "Indicate if fetch the latest config of tools") _ = cmd.RegisterFlagCompletionFunc("components", common.ArrayCompletion("DevOps")) return @@ -70,6 +73,9 @@ func (o *kindOption) reset(cmd *cobra.Command, args []string) (err error) { func (o *kindOption) preRunE(_ *cobra.Command, _ []string) (err error) { is := installer.Installer{ Provider: "github", + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Fetch: o.fetch, } err = is.CheckDepAndInstall(map[string]string{ "kind": "kind", @@ -224,6 +230,7 @@ type kindOption struct { portMappings map[string]string ksVersion string components []string + fetch bool Reset bool Nightly string diff --git a/kubectl-plugin/install/kk.go b/kubectl-plugin/install/kk.go index 1727a6f..14a17ff 100644 --- a/kubectl-plugin/install/kk.go +++ b/kubectl-plugin/install/kk.go @@ -2,17 +2,22 @@ package install import ( "fmt" + "github.com/AlecAivazis/survey/v2" + "github.com/linuxsuren/http-downloader/pkg/exec" "github.com/linuxsuren/http-downloader/pkg/installer" + "github.com/linuxsuren/http-downloader/pkg/net" "github.com/linuxsuren/ks/kubectl-plugin/common" + "github.com/linuxsuren/ks/kubectl-plugin/install/containerd" + "github.com/linuxsuren/ks/kubectl-plugin/types" "github.com/spf13/cobra" + "html/template" + "io/ioutil" + "os" + "path/filepath" + "runtime" "strings" ) -const ( - // DefaultKubeSphereVersion is the default version of KubeSphere - DefaultKubeSphereVersion = "v3.1.0" -) - func newInstallWithKKCmd() (cmd *cobra.Command) { opt := &kkOption{} cmd = &cobra.Command{ @@ -29,14 +34,16 @@ ks install kk --version nightly --components devops`, } flags := cmd.Flags() - flags.StringVarP(&opt.version, "version", "v", DefaultKubeSphereVersion, - fmt.Sprintf("The version of KubeSphere. Support value could be %s, nightly, nightly-20210309. nightly equals to nightly-latest", DefaultKubeSphereVersion)) + flags.StringVarP(&opt.version, "version", "v", types.KsVersion, + fmt.Sprintf("The version of KubeSphere. Support value could be %s, nightly, nightly-20210309. nightly equals to nightly-latest", types.KsVersion)) flags.StringArrayVarP(&opt.components, "components", "", []string{}, "The components which you want to enable after the installation") flags.StringVarP(&opt.zone, "zone", "", "cn", "Set environment variables, for example export KKZONE=cn") flags.StringVarP(&opt.container, "container", "", "docker", "Indicate the container runtime type. Supported: docker, containerd") + flags.BoolVarP(&opt.fetch, "fetch", "", true, + "Indicate if fetch the latest config of tools") return } @@ -46,6 +53,7 @@ type kkOption struct { components []string zone string container string + fetch bool } func (o *kkOption) versionCheck() (err error) { @@ -61,9 +69,9 @@ func (o *kkOption) versionCheck() (err error) { } else { o.version = ver } - } else if o.version != DefaultKubeSphereVersion { + } else if o.version != types.KsVersion { switch o.version { - case DefaultKubeSphereVersion, "v3.0.0": + case types.KsVersion, "v3.0.0": default: err = fmt.Errorf("not support version: %s", o.version) } @@ -76,11 +84,16 @@ func (o *kkOption) preRunE(cmd *cobra.Command, args []string) (err error) { return } - is := installer.Installer{ + is := &installer.Installer{ Provider: "github", + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Fetch: o.fetch, } dep := map[string]string{ - "kk": "kubesphere/kubekey", + "kk": "kubesphere/kubekey", + "socat": "socat", + "conntrack": "conntrack", } switch o.container { case "docker": @@ -90,7 +103,64 @@ func (o *kkOption) preRunE(cmd *cobra.Command, args []string) (err error) { dep["crictl"] = "kubernetes-sigs/cri-tools" dep["runc"] = "opencontainers/runc" } - err = is.CheckDepAndInstall(dep) + if err = is.CheckDepAndInstall(dep); err == nil { + err = setDefaultConfigFiles() + } + + // TODO find a better way to restart service + if err == nil { + err = enableAndRestartService(o.container) + } + return +} + +func enableAndRestartService(service string) (err error) { + if err = exec.RunCommand("systemctl", "enable", service); err != nil { + err = fmt.Errorf("failed to enable service: %s, error: %v", service, err) + } else { + if err = exec.RunCommand("systemctl", "restart", service); err != nil { + err = fmt.Errorf("failed to restart service: %s, error: %v", service, err) + } + } + return +} + +func setDefaultConfigFiles() (err error) { + if err = setDefaultIfNotExist([]byte(containerd.GetConfigToml()), "/etc/containerd/config.toml"); err == nil { + err = setDefaultIfNotExist([]byte(containerd.GetCrictl()), "/etc/crictl.yaml") + } + + if err != nil { + return + } + + err = setDefaultIfNotExist([]byte(containerd.GetContainerdService()), "/etc/systemd/system/containerd.service") + return +} + +func setDefaultIfNotExist(data []byte, path string) (err error) { + parentDir := filepath.Dir(path) + if err = os.MkdirAll(parentDir, 0644); err != nil { + err = fmt.Errorf("failed to create directory: %s, error: %v", parentDir, err) + return + } + + if _, err = os.Stat(path); os.IsNotExist(err) { + err = ioutil.WriteFile(path, data, 0644) + } else { + prompt := &survey.Select{ + Message: fmt.Sprintf("If you want to overwrite the existing file '%s'.", path), + Options: []string{"yes", "no"}, + } + var choose string + if err = survey.AskOne(prompt, &choose); err != nil { + return + } + + if choose == "yes" { + err = ioutil.WriteFile(path, data, 0644) + } + } return } @@ -98,10 +168,20 @@ func (o *kkOption) runE(cmd *cobra.Command, args []string) (err error) { report := installReport{} report.init() + var configFile string + if configFile, err = getTemporaryConfigFile(o.container); err != nil { + err = fmt.Errorf("failed to get a temporary kk config file, error: %v", err) + return + } + + defer func(file string) { + _ = os.RemoveAll(file) + }(configFile) commander := Commander{ Env: []string{fmt.Sprintf("KKZONE=%s", o.zone)}, } - if err = commander.execCommand("kk", "create", "cluster", "--with-kubesphere", o.version, "--yes"); err != nil { + if err = commander.execCommand("kk", "create", "cluster", "--filename", configFile, + "--with-kubesphere", o.version, "--yes"); err != nil { return } @@ -110,7 +190,37 @@ func (o *kkOption) runE(cmd *cobra.Command, args []string) (err error) { return } } - report.end() return } + +func getTemporaryConfigFile(container string) (filePath string, err error) { + var ( + kkFile *os.File + tpl *template.Template + address string + ) + + if kkFile, err = os.CreateTemp(os.TempDir(), "kk-config"); err != nil { + err = fmt.Errorf("failed to create temporary file for kk config, error: %v", err) + return + } + + filePath = kkFile.Name() + if address, err = net.GetExternalIP(); err != nil { + err = fmt.Errorf("failed to get external IP, error: %v", err) + return + } + + if tpl, err = template.New("config").Parse(containerd.GetKKConfig()); err == nil { + if err = tpl.Execute(kkFile, map[string]string{ + "container": container, + "address": address, + }); err != nil { + err = fmt.Errorf("failed to render kk config file template, error: %v", err) + } + } else { + err = fmt.Errorf("failed to create kk config file template, error: %v", err) + } + return +} diff --git a/kubectl-plugin/install/kk_test.go b/kubectl-plugin/install/kk_test.go index 82c8286..3e1152f 100644 --- a/kubectl-plugin/install/kk_test.go +++ b/kubectl-plugin/install/kk_test.go @@ -2,7 +2,11 @@ package install import ( "fmt" + "github.com/linuxsuren/ks/kubectl-plugin/types" "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "path/filepath" "testing" "time" ) @@ -14,8 +18,8 @@ func TestVersionCheck(t *testing.T) { returnErr bool message string }{{ - param: DefaultKubeSphereVersion, - expect: DefaultKubeSphereVersion, + param: types.KsVersion, + expect: types.KsVersion, returnErr: false, message: "do nothing with the default version", }, { @@ -57,3 +61,17 @@ func TestVersionCheck(t *testing.T) { } } } + +func TestSetDefaultIfNotExist(t *testing.T) { + data := []byte("data") + file := filepath.Join(os.TempDir(), "fake") + defer func(filePath string) { + _ = os.RemoveAll(filePath) + }(file) + + err := setDefaultIfNotExist(data, file) + assert.Nil(t, err, "failed to setDefaultIfNotExist") + gotData, err := ioutil.ReadFile(file) + assert.Nil(t, err, "failed to get the got data file") + assert.Equal(t, string(data), string(gotData)) +}