From 7e4ea379c99694a0a1cd331b3d40f5e8fbec438a Mon Sep 17 00:00:00 2001 From: zyy17 Date: Thu, 3 Aug 2023 20:56:26 +0800 Subject: [PATCH 1/5] refactor: move file operations from artifacts manager to utils package Signed-off-by: zyy17 --- pkg/deployer/baremetal/artifacts.go | 174 +++----------------------- pkg/utils/testdata/test-tar-gz.tar.gz | Bin 0 -> 160 bytes pkg/utils/testdata/test-tgz.tgz | Bin 0 -> 160 bytes pkg/utils/testdata/test-zip.zip | Bin 0 -> 332 bytes pkg/utils/utils.go | 145 +++++++++++++++++++++ pkg/utils/utils_test.go | 59 +++++++++ 6 files changed, 222 insertions(+), 156 deletions(-) create mode 100644 pkg/utils/testdata/test-tar-gz.tar.gz create mode 100644 pkg/utils/testdata/test-tgz.tgz create mode 100644 pkg/utils/testdata/test-zip.zip create mode 100644 pkg/utils/utils_test.go diff --git a/pkg/deployer/baremetal/artifacts.go b/pkg/deployer/baremetal/artifacts.go index 39da6243..82f3ec79 100644 --- a/pkg/deployer/baremetal/artifacts.go +++ b/pkg/deployer/baremetal/artifacts.go @@ -15,10 +15,6 @@ package baremetal import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" "context" "fmt" "io" @@ -41,12 +37,6 @@ const ( EtcdGitHubOrg = "etcd-io" EtcdGithubRepo = "etcd" - ZipExtension = ".zip" - TarGzExtension = ".tar.gz" - TarExtension = ".tar" - TgzExtension = ".tgz" - GzExtension = ".gz" - GOOSDarwin = "darwin" GOOSLinux = "linux" ) @@ -147,7 +137,7 @@ func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType } func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) error { - if err := am.uncompress(artifactFile, pkgDir); err != nil { + if err := utils.Uncompress(artifactFile, pkgDir); err != nil { return err } @@ -158,10 +148,10 @@ func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) erro artifactFile = path.Base(artifactFile) // If the artifactFile is '${pkgDir}/etcd-v3.5.7-darwin-arm64.zip', it will get '${pkgDir}/etcd-v3.5.7-darwin-arm64'. uncompressedDir := path.Join(pkgDir, artifactFile[:len(artifactFile)-len(filepath.Ext(artifactFile))]) - uncompressedDir = strings.TrimSuffix(uncompressedDir, TarExtension) + uncompressedDir = strings.TrimSuffix(uncompressedDir, utils.TarExtension) binaries := []string{"etcd", "etcdctl", "etcdutl"} for _, binary := range binaries { - if err := am.copyFile(path.Join(uncompressedDir, binary), path.Join(binDir, binary)); err != nil { + if err := utils.CopyFile(path.Join(uncompressedDir, binary), path.Join(binDir, binary)); err != nil { return err } if err := os.Chmod(path.Join(binDir, binary), 0755); err != nil { @@ -176,7 +166,7 @@ func (am *ArtifactManager) installGreptime(artifactFile, binDir string) error { return err } - if err := am.uncompress(artifactFile, binDir); err != nil { + if err := utils.Uncompress(artifactFile, binDir); err != nil { return err } @@ -188,16 +178,7 @@ func (am *ArtifactManager) installGreptime(artifactFile, binDir string) error { } func (am *ArtifactManager) download(ctx context.Context, typ ArtifactType, version, pkgDir string) (string, error) { - var extension string - switch runtime.GOOS { - case GOOSDarwin: - extension = ZipExtension - case GOOSLinux: - extension = TarGzExtension - default: - return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) - } - downloadURL, err := am.artifactURL(typ, version, extension) + downloadURL, err := am.artifactURL(typ, version) if err != nil { return "", err } @@ -255,7 +236,18 @@ func (am *ArtifactManager) download(ctx context.Context, typ ArtifactType, versi return artifactFile, nil } -func (am *ArtifactManager) artifactURL(typ ArtifactType, version, ext string) (string, error) { +func (am *ArtifactManager) artifactURL(typ ArtifactType, version string) (string, error) { + var etcdPackageExt string + + switch runtime.GOOS { + case GOOSDarwin: + etcdPackageExt = utils.ZipExtension + case GOOSLinux: + etcdPackageExt = utils.TarGzExtension + default: + return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) + } + switch typ { case GreptimeArtifactType: var downloadURL string @@ -270,139 +262,9 @@ func (am *ArtifactManager) artifactURL(typ ArtifactType, version, ext string) (s case EtcdArtifactType: // For the function stability, we use the specific version of etcd. downloadURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s%s", - EtcdGitHubOrg, EtcdGithubRepo, version, string(typ), version, runtime.GOOS, runtime.GOARCH, ext) + EtcdGitHubOrg, EtcdGithubRepo, version, string(typ), version, runtime.GOOS, runtime.GOARCH, etcdPackageExt) return downloadURL, nil default: return "", fmt.Errorf("unsupported artifact type: %v", typ) } } - -func (am *ArtifactManager) uncompress(file, dst string) error { - fileType := path.Ext(file) - switch fileType { - case ZipExtension: - return am.unzip(file, dst) - case TgzExtension: - return am.untar(file, dst) - case GzExtension: - return am.untar(file, dst) - default: - return fmt.Errorf("unsupported file type: %s", fileType) - } -} - -func (am *ArtifactManager) unzip(file, dst string) error { - archive, err := zip.OpenReader(file) - if err != nil { - return err - } - defer archive.Close() - - for _, f := range archive.File { - filePath := filepath.Join(dst, f.Name) - - if f.FileInfo().IsDir() { - if err := os.MkdirAll(filePath, os.ModePerm); err != nil { - return err - } - continue - } - - if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { - return err - } - - dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - - fileInArchive, err := f.Open() - if err != nil { - return err - } - - if _, err := io.Copy(dstFile, fileInArchive); err != nil { - return err - } - - if err := dstFile.Close(); err != nil { - return err - } - - if err := fileInArchive.Close(); err != nil { - return err - } - } - - return nil -} - -func (am *ArtifactManager) untar(file, dst string) error { - data, err := os.ReadFile(file) - if err != nil { - return err - } - - stream, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return err - } - - tarReader := tar.NewReader(stream) - - for { - header, err := tarReader.Next() - - if err == io.EOF { - break - } - - if err != nil { - return err - } - - switch header.Typeflag { - case tar.TypeReg: - outFile, err := os.Create(dst + "/" + header.Name) - if err != nil { - return err - } - if _, err := io.Copy(outFile, tarReader); err != nil { - return err - } - if err := outFile.Close(); err != nil { - return err - } - case tar.TypeDir: - if err := os.Mkdir(dst+"/"+header.Name, 0755); err != nil { - return err - } - default: - continue - } - } - - return nil -} - -func (am *ArtifactManager) copyFile(src, dst string) error { - r, err := os.Open(src) - if err != nil { - return err - } - defer r.Close() - - w, err := os.Create(dst) - if err != nil { - return err - } - defer w.Close() - - _, err = io.Copy(w, r) - if err != nil { - return err - } - - return w.Sync() -} diff --git a/pkg/utils/testdata/test-tar-gz.tar.gz b/pkg/utils/testdata/test-tar-gz.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..6d7f2dd49904f951a8cf58deb156c02582bb8fa9 GIT binary patch literal 160 zcmV;R0AK$fiwFQ^p37ta1MQJP3WG2ZMRSz7fOICABA&mEN3hlQyDT<;f O&h!8gAt@{X3IG7&drDjY literal 0 HcmV?d00001 diff --git a/pkg/utils/testdata/test-zip.zip b/pkg/utils/testdata/test-zip.zip new file mode 100644 index 0000000000000000000000000000000000000000..b934dde1d4c4316a23f3dc85d50b7f66b77aaabd GIT binary patch literal 332 zcmWIWW@h1H0D)x-nZv;hD8b1f!%&i1T%uc*S)d;p!pXpVW9I1;5H79YW?*Fb%E-XL zA_7z#fMUd2h1Wkp27)jzvJokXC5dojZekjfk(!f}U!Gr-lM>*~$Rx*%%T*Fk_X#k( xbp$by+{g-XBZm7>jYD-I#5kZ|7?w0TBO3>CFD@fl*+6zP0pTJbJq5&J002=ML-_yz literal 0 HcmV?d00001 diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index d5d28b69..102f7f99 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -15,8 +15,15 @@ package utils import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" "fmt" + "io" "os" + "path" + "path/filepath" ) func CreateDirIfNotExists(dir string) (err error) { @@ -53,3 +60,141 @@ func IsFileExists(filepath string) (bool, error) { // The file exists. return true, nil } + +// CopyFile copies the file from src to dst. +func CopyFile(src, dst string) error { + r, err := os.Open(src) + if err != nil { + return err + } + defer r.Close() + + w, err := os.Create(dst) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + if err != nil { + return err + } + + return w.Sync() +} + +const ( + ZipExtension = ".zip" + TarGzExtension = ".tar.gz" + TgzExtension = ".tgz" + GzExtension = ".gz" + TarExtension = ".tar" +) + +// Uncompress uncompresses the file to the destination directory. +func Uncompress(file, dst string) error { + fileType := path.Ext(file) + switch fileType { + case ZipExtension: + return unzip(file, dst) + case TgzExtension, GzExtension, TarGzExtension: + return untar(file, dst) + default: + return fmt.Errorf("unsupported file type: %s", fileType) + } +} + +func unzip(file, dst string) error { + archive, err := zip.OpenReader(file) + if err != nil { + return err + } + defer archive.Close() + + for _, f := range archive.File { + filePath := filepath.Join(dst, f.Name) + + if f.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, os.ModePerm); err != nil { + return err + } + continue + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return err + } + + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + + fileInArchive, err := f.Open() + if err != nil { + return err + } + + if _, err := io.Copy(dstFile, fileInArchive); err != nil { + return err + } + + if err := dstFile.Close(); err != nil { + return err + } + + if err := fileInArchive.Close(); err != nil { + return err + } + } + + return nil +} + +func untar(file, dst string) error { + data, err := os.ReadFile(file) + if err != nil { + return err + } + + stream, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + + tarReader := tar.NewReader(stream) + + for { + header, err := tarReader.Next() + + if err == io.EOF { + break + } + + if err != nil { + return err + } + + switch header.Typeflag { + case tar.TypeReg: + outFile, err := os.Create(dst + "/" + header.Name) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tarReader); err != nil { + return err + } + if err := outFile.Close(); err != nil { + return err + } + case tar.TypeDir: + if err := os.Mkdir(dst+"/"+header.Name, 0755); err != nil { + return err + } + default: + continue + } + } + + return nil +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 00000000..2e71eca1 --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,59 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" + "path" + "testing" +) + +func TestUncompress(t *testing.T) { + const ( + testContent = "helloworld" + outputDir = "testdata/output" + ) + + tests := []struct { + filename string + path string + dst string + }{ + {"test-zip", "testdata/test-zip.zip", outputDir}, + {"test-tgz", "testdata/test-tgz.tgz", outputDir}, + {"test-tgz", "testdata/test-tar-gz.tar.gz", outputDir}, + } + + // Clean up output dir. + defer func() { + os.RemoveAll(outputDir) + }() + + for _, test := range tests { + if err := Uncompress(test.path, test.dst); err != nil { + t.Errorf("uncompress file '%s': %v", test.path, err) + } + + dataFile := path.Join(test.dst, test.filename, "data") + data, err := os.ReadFile(dataFile) + if err != nil { + t.Errorf("read file '%s': %v", dataFile, err) + } + + if string(data) != testContent { + t.Errorf("file content is not '%s': %s", testContent, string(data)) + } + } +} From 9998cfcdbfa717b6d0366bba3056f9823669fff9 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Fri, 4 Aug 2023 20:37:06 +0800 Subject: [PATCH 2/5] refactor: adapt to the upstream release breaking changes --- go.mod | 16 +-- go.sum | 33 +++++-- pkg/deployer/baremetal/artifacts.go | 118 +++++++++++++++++------ pkg/deployer/baremetal/artifacts_test.go | 36 +++++++ pkg/utils/utils.go | 18 ++++ 5 files changed, 178 insertions(+), 43 deletions(-) create mode 100644 pkg/deployer/baremetal/artifacts_test.go diff --git a/go.mod b/go.mod index 288f3416..a93cd112 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.18 require ( github.com/GreptimeTeam/greptimedb-operator v0.1.0-alpha.9 github.com/Masterminds/semver v1.5.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/briandowns/spinner v1.19.0 github.com/fatih/color v1.13.0 github.com/go-playground/validator/v10 v10.14.1 github.com/go-sql-driver/mysql v1.6.0 github.com/google/go-cmp v0.5.9 + github.com/google/go-github/v53 v53.2.0 github.com/onsi/ginkgo/v2 v2.4.0 github.com/onsi/gomega v1.23.0 github.com/spf13/cobra v1.6.1 @@ -31,13 +33,14 @@ require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/containerd/containerd v1.6.18 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -65,6 +68,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.2.0 // indirect @@ -120,12 +124,12 @@ require ( github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect diff --git a/go.sum b/go.sum index 428d204b..4f03ae92 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,9 @@ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy86 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.0/go.mod h1:tWhwTbUTndesPNeF0C900vKoq283u6zp4APT9vaF3SI= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= @@ -64,6 +65,8 @@ github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvd github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -90,6 +93,7 @@ github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuP github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -100,6 +104,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -271,6 +278,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= +github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -702,8 +713,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -717,8 +728,9 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,6 +804,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -799,13 +812,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -816,8 +829,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/deployer/baremetal/artifacts.go b/pkg/deployer/baremetal/artifacts.go index 82f3ec79..b1f6e919 100644 --- a/pkg/deployer/baremetal/artifacts.go +++ b/pkg/deployer/baremetal/artifacts.go @@ -28,17 +28,21 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" "github.com/GreptimeTeam/gtctl/pkg/utils" + "github.com/google/go-github/v53/github" ) const ( GreptimeGitHubOrg = "GreptimeTeam" GreptimeDBGithubRepo = "greptimedb" + GreptimeBinName = "greptime" EtcdGitHubOrg = "etcd-io" EtcdGithubRepo = "etcd" GOOSDarwin = "darwin" GOOSLinux = "linux" + + BreakingChangeVersion = "v0.4.0-nightly-20230802" ) // ArtifactManager is responsible for managing the artifacts of a GreptimeDB cluster. @@ -92,12 +96,24 @@ func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType return nil } + version := artifact.Version + + // Get the latest greptime released version. + if typ == GreptimeArtifactType && artifact.Version == "latest" { + client := github.NewClient(nil) + release, _, err := client.Repositories.GetLatestRelease(ctx, GreptimeGitHubOrg, GreptimeDBGithubRepo) + if err != nil { + return err + } + version = *release.TagName + } + var ( - pkgDir = path.Join(am.dir, typ.String(), artifact.Version, "pkg") - binDir = path.Join(am.dir, typ.String(), artifact.Version, "bin") + pkgDir = path.Join(am.dir, typ.String(), version, "pkg") + binDir = path.Join(am.dir, typ.String(), version, "bin") ) - artifactFile, err := am.download(ctx, typ, artifact.Version, pkgDir) + artifactFile, err := am.download(ctx, typ, version, pkgDir) if err != nil { return err } @@ -128,7 +144,7 @@ func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType // └── greptime-darwin-arm64.tgz switch typ { case GreptimeArtifactType: - return am.installGreptime(artifactFile, binDir) + return am.installGreptime(artifactFile, binDir, version) case EtcdArtifactType: return am.installEtcd(artifactFile, pkgDir, binDir) default: @@ -161,7 +177,7 @@ func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) erro return nil } -func (am *ArtifactManager) installGreptime(artifactFile, binDir string) error { +func (am *ArtifactManager) installGreptime(artifactFile, binDir, version string) error { if err := utils.CreateDirIfNotExists(binDir); err != nil { return err } @@ -170,7 +186,23 @@ func (am *ArtifactManager) installGreptime(artifactFile, binDir string) error { return err } - if err := os.Chmod(path.Join(binDir, "greptime"), 0755); err != nil { + newVersion, err := am.isBreakingVersion(version) + if err != nil { + return err + } + + // If it's the breaking version, adapt to the new directory layout. + if newVersion { + originalBinDir := path.Join(binDir, strings.TrimSuffix(path.Base(artifactFile), utils.TarGzExtension)) + if err := os.Rename(path.Join(originalBinDir, GreptimeBinName), path.Join(binDir, GreptimeBinName)); err != nil { + return err + } + if err := os.Remove(originalBinDir); err != nil { + return err + } + } + + if err := os.Chmod(path.Join(binDir, GreptimeBinName), 0755); err != nil { return err } @@ -237,34 +269,66 @@ func (am *ArtifactManager) download(ctx context.Context, typ ArtifactType, versi } func (am *ArtifactManager) artifactURL(typ ArtifactType, version string) (string, error) { - var etcdPackageExt string + switch typ { + case GreptimeArtifactType: + return am.greptimeDownloadURL(version) + case EtcdArtifactType: + return am.etcdDownloadURL(version) + default: + return "", fmt.Errorf("unsupported artifact type: %v", typ) + } +} + +func (am *ArtifactManager) getGreptimeLatestVersion() (string, error) { + client := github.NewClient(nil) + release, _, err := client.Repositories.GetLatestRelease(context.Background(), GreptimeGitHubOrg, GreptimeDBGithubRepo) + if err != nil { + return "", err + } + return *release.TagName, nil +} + +func (am *ArtifactManager) greptimeDownloadURL(version string) (string, error) { + newVersion, err := am.isBreakingVersion(version) + if err != nil { + return "", err + } + + // If version >= BreakingChangeVersion, use the new download URL. + if newVersion { + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s.tar.gz", + GreptimeGitHubOrg, GreptimeDBGithubRepo, version, string(GreptimeArtifactType), runtime.GOOS, runtime.GOARCH, version), nil + } + + // If version < BreakingChangeVersion, use the old download URL. + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s.tgz", + GreptimeGitHubOrg, GreptimeDBGithubRepo, version, string(GreptimeArtifactType), runtime.GOOS, runtime.GOARCH), nil +} + +func (am *ArtifactManager) etcdDownloadURL(version string) (string, error) { + var ext string switch runtime.GOOS { case GOOSDarwin: - etcdPackageExt = utils.ZipExtension + ext = utils.ZipExtension case GOOSLinux: - etcdPackageExt = utils.TarGzExtension + ext = utils.TarGzExtension default: return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) } - switch typ { - case GreptimeArtifactType: - var downloadURL string - if version == "latest" { - downloadURL = fmt.Sprintf("https://github.com/%s/%s/releases/latest/download/%s-%s-%s.tgz", - GreptimeGitHubOrg, GreptimeDBGithubRepo, string(typ), runtime.GOOS, runtime.GOARCH) - } else { - downloadURL = fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s.tgz", - GreptimeGitHubOrg, GreptimeDBGithubRepo, version, string(typ), runtime.GOOS, runtime.GOARCH) - } - return downloadURL, nil - case EtcdArtifactType: - // For the function stability, we use the specific version of etcd. - downloadURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s%s", - EtcdGitHubOrg, EtcdGithubRepo, version, string(typ), version, runtime.GOOS, runtime.GOARCH, etcdPackageExt) - return downloadURL, nil - default: - return "", fmt.Errorf("unsupported artifact type: %v", typ) + // For the function stability, we use the specific version of etcd. + downloadURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s%s", + EtcdGitHubOrg, EtcdGithubRepo, version, string(EtcdArtifactType), version, runtime.GOOS, runtime.GOARCH, ext) + + return downloadURL, nil +} + +func (am *ArtifactManager) isBreakingVersion(version string) (bool, error) { + newVersion, err := utils.SemVerCompare(version, BreakingChangeVersion) + if err != nil { + return false, err } + + return newVersion || version == BreakingChangeVersion, nil } diff --git a/pkg/deployer/baremetal/artifacts_test.go b/pkg/deployer/baremetal/artifacts_test.go new file mode 100644 index 00000000..1818c5f3 --- /dev/null +++ b/pkg/deployer/baremetal/artifacts_test.go @@ -0,0 +1,36 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baremetal + +import ( + "context" + "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + "github.com/GreptimeTeam/gtctl/pkg/logger" + "os" + "sigs.k8s.io/kind/pkg/log" + "testing" +) + +func TestArtifactManager(t *testing.T) { + am, err := NewArtifactManager("/tmp/gtctl-test-am", logger.New(os.Stdout, log.Level(4), logger.WithColored()), false) + if err != nil { + t.Errorf("failed to create artifact manager: %v", err) + } + + ctx := context.Background() + if err := am.PrepareArtifact(ctx, GreptimeArtifactType, &config.Artifact{Version: "latest"}); err != nil { + t.Errorf("failed to prepare latest greptime artifact: %v", err) + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 102f7f99..4907d938 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -24,6 +24,8 @@ import ( "os" "path" "path/filepath" + + "github.com/Masterminds/semver/v3" ) func CreateDirIfNotExists(dir string) (err error) { @@ -83,6 +85,22 @@ func CopyFile(src, dst string) error { return w.Sync() } +// SemVerCompare compares two semantic versions. +// It returns true if v1 is greater than v2, otherwise false. +func SemVerCompare(v1, v2 string) (bool, error) { + semV1, err := semver.NewVersion(v1) + if err != nil { + return false, err + } + + semV2, err := semver.NewVersion(v2) + if err != nil { + return false, err + } + + return semV1.GreaterThan(semV2), nil +} + const ( ZipExtension = ".zip" TarGzExtension = ".tar.gz" From bdafc7be6913a935468b5d85f6a2c3bd0e19083b Mon Sep 17 00:00:00 2001 From: zyy17 Date: Mon, 7 Aug 2023 15:31:23 +0800 Subject: [PATCH 3/5] style: refactor structure of 'pkg/utils' Signed-off-by: zyy17 --- pkg/deployer/baremetal/artifacts.go | 27 +++++++------- pkg/deployer/baremetal/component/datanode.go | 18 ++++----- pkg/deployer/baremetal/component/etcd.go | 6 +-- pkg/deployer/baremetal/component/frontend.go | 10 ++--- pkg/deployer/baremetal/component/metasrv.go | 6 +-- pkg/deployer/baremetal/deployer.go | 10 ++--- pkg/utils/{utils.go => file/file.go} | 20 +--------- .../{utils_test.go => file/file_test.go} | 2 +- .../{ => file}/testdata/test-tar-gz.tar.gz | Bin pkg/utils/{ => file}/testdata/test-tgz.tgz | Bin pkg/utils/{ => file}/testdata/test-zip.zip | Bin pkg/utils/semver/semver.go | 35 ++++++++++++++++++ 12 files changed, 76 insertions(+), 58 deletions(-) rename pkg/utils/{utils.go => file/file.go} (90%) rename pkg/utils/{utils_test.go => file/file_test.go} (99%) rename pkg/utils/{ => file}/testdata/test-tar-gz.tar.gz (100%) rename pkg/utils/{ => file}/testdata/test-tgz.tgz (100%) rename pkg/utils/{ => file}/testdata/test-zip.zip (100%) create mode 100644 pkg/utils/semver/semver.go diff --git a/pkg/deployer/baremetal/artifacts.go b/pkg/deployer/baremetal/artifacts.go index b1f6e919..85776b00 100644 --- a/pkg/deployer/baremetal/artifacts.go +++ b/pkg/deployer/baremetal/artifacts.go @@ -27,7 +27,8 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" + semverutils "github.com/GreptimeTeam/gtctl/pkg/utils/semver" "github.com/google/go-github/v53/github" ) @@ -69,7 +70,7 @@ func (t ArtifactType) String() string { func NewArtifactManager(workingDir string, l logger.Logger, alwaysDownload bool) (*ArtifactManager, error) { dir := path.Join(workingDir, "artifacts") - if err := utils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.CreateDirIfNotExists(dir); err != nil { return nil, err } @@ -153,21 +154,21 @@ func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType } func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) error { - if err := utils.Uncompress(artifactFile, pkgDir); err != nil { + if err := fileutils.Uncompress(artifactFile, pkgDir); err != nil { return err } - if err := utils.CreateDirIfNotExists(binDir); err != nil { + if err := fileutils.CreateDirIfNotExists(binDir); err != nil { return err } artifactFile = path.Base(artifactFile) // If the artifactFile is '${pkgDir}/etcd-v3.5.7-darwin-arm64.zip', it will get '${pkgDir}/etcd-v3.5.7-darwin-arm64'. uncompressedDir := path.Join(pkgDir, artifactFile[:len(artifactFile)-len(filepath.Ext(artifactFile))]) - uncompressedDir = strings.TrimSuffix(uncompressedDir, utils.TarExtension) + uncompressedDir = strings.TrimSuffix(uncompressedDir, fileutils.TarExtension) binaries := []string{"etcd", "etcdctl", "etcdutl"} for _, binary := range binaries { - if err := utils.CopyFile(path.Join(uncompressedDir, binary), path.Join(binDir, binary)); err != nil { + if err := fileutils.CopyFile(path.Join(uncompressedDir, binary), path.Join(binDir, binary)); err != nil { return err } if err := os.Chmod(path.Join(binDir, binary), 0755); err != nil { @@ -178,11 +179,11 @@ func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) erro } func (am *ArtifactManager) installGreptime(artifactFile, binDir, version string) error { - if err := utils.CreateDirIfNotExists(binDir); err != nil { + if err := fileutils.CreateDirIfNotExists(binDir); err != nil { return err } - if err := utils.Uncompress(artifactFile, binDir); err != nil { + if err := fileutils.Uncompress(artifactFile, binDir); err != nil { return err } @@ -193,7 +194,7 @@ func (am *ArtifactManager) installGreptime(artifactFile, binDir, version string) // If it's the breaking version, adapt to the new directory layout. if newVersion { - originalBinDir := path.Join(binDir, strings.TrimSuffix(path.Base(artifactFile), utils.TarGzExtension)) + originalBinDir := path.Join(binDir, strings.TrimSuffix(path.Base(artifactFile), fileutils.TarGzExtension)) if err := os.Rename(path.Join(originalBinDir, GreptimeBinName), path.Join(binDir, GreptimeBinName)); err != nil { return err } @@ -215,7 +216,7 @@ func (am *ArtifactManager) download(ctx context.Context, typ ArtifactType, versi return "", err } - if err := utils.CreateDirIfNotExists(pkgDir); err != nil { + if err := fileutils.CreateDirIfNotExists(pkgDir); err != nil { return "", err } @@ -310,9 +311,9 @@ func (am *ArtifactManager) etcdDownloadURL(version string) (string, error) { switch runtime.GOOS { case GOOSDarwin: - ext = utils.ZipExtension + ext = fileutils.ZipExtension case GOOSLinux: - ext = utils.TarGzExtension + ext = fileutils.TarGzExtension default: return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) } @@ -325,7 +326,7 @@ func (am *ArtifactManager) etcdDownloadURL(version string) (string, error) { } func (am *ArtifactManager) isBreakingVersion(version string) (bool, error) { - newVersion, err := utils.SemVerCompare(version, BreakingChangeVersion) + newVersion, err := semverutils.Compare(version, BreakingChangeVersion) if err != nil { return false, err } diff --git a/pkg/deployer/baremetal/component/datanode.go b/pkg/deployer/baremetal/component/datanode.go index a8c01282..1468083a 100644 --- a/pkg/deployer/baremetal/component/datanode.go +++ b/pkg/deployer/baremetal/component/datanode.go @@ -26,7 +26,7 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) type datanode struct { @@ -55,7 +55,7 @@ func newDataNodes(config *config.Datanode, metaSrvAddr string, workDirs WorkDirs func (d *datanode) Start(ctx context.Context, binary string) error { dataHome := path.Join(d.workDirs.DataDir, "home") - if err := utils.CreateDirIfNotExists(dataHome); err != nil { + if err := fileutils.CreateDirIfNotExists(dataHome); err != nil { return err } d.dataHome = dataHome @@ -64,19 +64,19 @@ func (d *datanode) Start(ctx context.Context, binary string) error { dirName := fmt.Sprintf("datanode.%d", i) datanodeLogDir := path.Join(d.workDirs.LogsDir, dirName) - if err := utils.CreateDirIfNotExists(datanodeLogDir); err != nil { + if err := fileutils.CreateDirIfNotExists(datanodeLogDir); err != nil { return err } d.dataNodeLogDirs = append(d.dataNodeLogDirs, datanodeLogDir) datanodePidDir := path.Join(d.workDirs.PidsDir, dirName) - if err := utils.CreateDirIfNotExists(datanodePidDir); err != nil { + if err := fileutils.CreateDirIfNotExists(datanodePidDir); err != nil { return err } d.dataNodePidDirs = append(d.dataNodePidDirs, datanodePidDir) walDir := path.Join(d.workDirs.DataDir, dirName, "wal") - if err := utils.CreateDirIfNotExists(walDir); err != nil { + if err := fileutils.CreateDirIfNotExists(walDir); err != nil { return err } d.dataNodeDataDirs = append(d.dataNodeDataDirs, path.Join(d.workDirs.DataDir, dirName)) @@ -155,24 +155,24 @@ func (d *datanode) IsRunning(ctx context.Context) bool { } func (d *datanode) Delete(ctx context.Context) error { - if err := utils.DeleteDirIfExists(d.dataHome); err != nil { + if err := fileutils.DeleteDirIfExists(d.dataHome); err != nil { return err } for _, dir := range d.dataNodeLogDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } for _, dir := range d.dataNodePidDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } for _, dir := range d.dataNodeDataDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } diff --git a/pkg/deployer/baremetal/component/etcd.go b/pkg/deployer/baremetal/component/etcd.go index 8df856eb..6d4043e3 100644 --- a/pkg/deployer/baremetal/component/etcd.go +++ b/pkg/deployer/baremetal/component/etcd.go @@ -20,7 +20,7 @@ import ( "sync" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) type etcd struct { @@ -47,7 +47,7 @@ func (e *etcd) Start(ctx context.Context, binary string) error { etcdDirs = []string{etcdDataDir, etcdLogDir, etcdPidDir} ) for _, dir := range etcdDirs { - if err := utils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.CreateDirIfNotExists(dir); err != nil { return err } } @@ -71,7 +71,7 @@ func (e *etcd) IsRunning(ctx context.Context) bool { func (e *etcd) Delete(ctx context.Context) error { for _, dir := range e.etcdDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } diff --git a/pkg/deployer/baremetal/component/frontend.go b/pkg/deployer/baremetal/component/frontend.go index 5aa13238..d80d356b 100644 --- a/pkg/deployer/baremetal/component/frontend.go +++ b/pkg/deployer/baremetal/component/frontend.go @@ -22,7 +22,7 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) type frontend struct { @@ -52,13 +52,13 @@ func (f *frontend) Start(ctx context.Context, binary string) error { dirName := fmt.Sprintf("frontend.%d", i) frontendLogDir := path.Join(f.workDirs.LogsDir, dirName) - if err := utils.CreateDirIfNotExists(frontendLogDir); err != nil { + if err := fileutils.CreateDirIfNotExists(frontendLogDir); err != nil { return err } f.frontendLogDirs = append(f.frontendLogDirs, frontendLogDir) frontendPidDir := path.Join(f.workDirs.PidsDir, dirName) - if err := utils.CreateDirIfNotExists(frontendPidDir); err != nil { + if err := fileutils.CreateDirIfNotExists(frontendPidDir); err != nil { return err } f.frontendPidDirs = append(f.frontendPidDirs, frontendPidDir) @@ -91,13 +91,13 @@ func (f *frontend) IsRunning(ctx context.Context) bool { func (f *frontend) Delete(ctx context.Context) error { for _, dir := range f.frontendLogDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } for _, dir := range f.frontendPidDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } diff --git a/pkg/deployer/baremetal/component/metasrv.go b/pkg/deployer/baremetal/component/metasrv.go index 6ccdfc3c..1e4cca72 100644 --- a/pkg/deployer/baremetal/component/metasrv.go +++ b/pkg/deployer/baremetal/component/metasrv.go @@ -25,7 +25,7 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) type metaSrv struct { @@ -54,7 +54,7 @@ func (m *metaSrv) Start(ctx context.Context, binary string) error { metaSrvDirs = []string{metaSrvLogDir, metaSrvPidDir} ) for _, dir := range metaSrvDirs { - if err := utils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.CreateDirIfNotExists(dir); err != nil { return err } } @@ -119,7 +119,7 @@ func (m *metaSrv) IsRunning(ctx context.Context) bool { func (m *metaSrv) Delete(ctx context.Context) error { for _, dir := range m.metaSrvDirs { - if err := utils.DeleteDirIfExists(dir); err != nil { + if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } diff --git a/pkg/deployer/baremetal/deployer.go b/pkg/deployer/baremetal/deployer.go index 2835243b..09089004 100644 --- a/pkg/deployer/baremetal/deployer.go +++ b/pkg/deployer/baremetal/deployer.go @@ -28,7 +28,7 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/utils" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) type Deployer struct { @@ -72,7 +72,7 @@ func NewDeployer(l logger.Logger, clusterName string, opts ...Option) (Interface return nil, err } d.workingDir = path.Join(homeDir, config.GtctlDir) - if err := utils.CreateDirIfNotExists(d.workingDir); err != nil { + if err := fileutils.CreateDirIfNotExists(d.workingDir); err != nil { return nil, err } @@ -121,7 +121,7 @@ func (d *Deployer) createClusterDirs(clusterName string) error { } for _, dir := range dirs { - if err := utils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.CreateDirIfNotExists(dir); err != nil { return err } } @@ -198,7 +198,7 @@ func (d *Deployer) DeleteGreptimeDBCluster(ctx context.Context, name string, opt func (d *Deployer) deleteGreptimeDBClusterForeground(ctx context.Context) error { // It is unnecessary to delete each component resources in cluster since it runs in the foreground. // So deleting the whole cluster resources here would be fine. - if err := utils.DeleteDirIfExists(d.clusterDir); err != nil { + if err := fileutils.DeleteDirIfExists(d.clusterDir); err != nil { return err } @@ -229,7 +229,7 @@ func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, op func (d *Deployer) checkEtcdHealth(etcdBin string) error { // It's very likely that "etcdctl" is under the same directory of "etcd". etcdctlBin := path.Join(etcdBin, "../etcdctl") - exists, err := utils.IsFileExists(etcdctlBin) + exists, err := fileutils.IsFileExists(etcdctlBin) if err != nil { return err } diff --git a/pkg/utils/utils.go b/pkg/utils/file/file.go similarity index 90% rename from pkg/utils/utils.go rename to pkg/utils/file/file.go index 4907d938..a020d989 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/file/file.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package file import ( "archive/tar" @@ -24,8 +24,6 @@ import ( "os" "path" "path/filepath" - - "github.com/Masterminds/semver/v3" ) func CreateDirIfNotExists(dir string) (err error) { @@ -85,22 +83,6 @@ func CopyFile(src, dst string) error { return w.Sync() } -// SemVerCompare compares two semantic versions. -// It returns true if v1 is greater than v2, otherwise false. -func SemVerCompare(v1, v2 string) (bool, error) { - semV1, err := semver.NewVersion(v1) - if err != nil { - return false, err - } - - semV2, err := semver.NewVersion(v2) - if err != nil { - return false, err - } - - return semV1.GreaterThan(semV2), nil -} - const ( ZipExtension = ".zip" TarGzExtension = ".tar.gz" diff --git a/pkg/utils/utils_test.go b/pkg/utils/file/file_test.go similarity index 99% rename from pkg/utils/utils_test.go rename to pkg/utils/file/file_test.go index 2e71eca1..0dd5e38f 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/file/file_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package file import ( "os" diff --git a/pkg/utils/testdata/test-tar-gz.tar.gz b/pkg/utils/file/testdata/test-tar-gz.tar.gz similarity index 100% rename from pkg/utils/testdata/test-tar-gz.tar.gz rename to pkg/utils/file/testdata/test-tar-gz.tar.gz diff --git a/pkg/utils/testdata/test-tgz.tgz b/pkg/utils/file/testdata/test-tgz.tgz similarity index 100% rename from pkg/utils/testdata/test-tgz.tgz rename to pkg/utils/file/testdata/test-tgz.tgz diff --git a/pkg/utils/testdata/test-zip.zip b/pkg/utils/file/testdata/test-zip.zip similarity index 100% rename from pkg/utils/testdata/test-zip.zip rename to pkg/utils/file/testdata/test-zip.zip diff --git a/pkg/utils/semver/semver.go b/pkg/utils/semver/semver.go new file mode 100644 index 00000000..50dce3fb --- /dev/null +++ b/pkg/utils/semver/semver.go @@ -0,0 +1,35 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package semver + +import ( + "github.com/Masterminds/semver/v3" +) + +// SemVerCompare compares two semantic versions. +// It returns true if v1 is greater than v2, otherwise false. +func Compare(v1, v2 string) (bool, error) { + semV1, err := semver.NewVersion(v1) + if err != nil { + return false, err + } + + semV2, err := semver.NewVersion(v2) + if err != nil { + return false, err + } + + return semV1.GreaterThan(semV2), nil +} From c46f23462e24fd0fa5ac078fa9f036c3553b98ab Mon Sep 17 00:00:00 2001 From: zyy17 Date: Mon, 7 Aug 2023 15:36:27 +0800 Subject: [PATCH 4/5] test: add 'semver_test.go' --- pkg/utils/semver/semver_test.go | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 pkg/utils/semver/semver_test.go diff --git a/pkg/utils/semver/semver_test.go b/pkg/utils/semver/semver_test.go new file mode 100644 index 00000000..5f62ed40 --- /dev/null +++ b/pkg/utils/semver/semver_test.go @@ -0,0 +1,40 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package semver + +import ( + "testing" +) + +func TestCompare(t *testing.T) { + tests := []struct { + v1, v2 string + want bool + }{ + {"v0.3.2", "v0.4.0-nightly-20230802", false}, + {"v0.4.0-nightly-20230807", "0.4.0-nightly-20230802", true}, + } + + for _, test := range tests { + got, err := Compare(test.v1, test.v2) + if err != nil { + t.Errorf("compare '%s' and '%s': %v", test.v1, test.v2, err) + } + + if got != test.want { + t.Errorf("compare '%s' and '%s': got %v, want %v", test.v1, test.v2, got, test.want) + } + } +} From b85f25cbf7fff690697a34b7ca18ae9c1b04673b Mon Sep 17 00:00:00 2001 From: zyy17 Date: Mon, 7 Aug 2023 16:07:04 +0800 Subject: [PATCH 5/5] test: refactor 'artifacts_test.go' Signed-off-by: zyy17 --- pkg/deployer/baremetal/artifacts.go | 35 ++++++++++++--------- pkg/deployer/baremetal/artifacts_test.go | 39 ++++++++++++++++++++---- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/pkg/deployer/baremetal/artifacts.go b/pkg/deployer/baremetal/artifacts.go index 85776b00..cf1e6180 100644 --- a/pkg/deployer/baremetal/artifacts.go +++ b/pkg/deployer/baremetal/artifacts.go @@ -97,21 +97,10 @@ func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType return nil } - version := artifact.Version - - // Get the latest greptime released version. - if typ == GreptimeArtifactType && artifact.Version == "latest" { - client := github.NewClient(nil) - release, _, err := client.Repositories.GetLatestRelease(ctx, GreptimeGitHubOrg, GreptimeDBGithubRepo) - if err != nil { - return err - } - version = *release.TagName - } - var ( - pkgDir = path.Join(am.dir, typ.String(), version, "pkg") - binDir = path.Join(am.dir, typ.String(), version, "bin") + version = artifact.Version + pkgDir = path.Join(am.dir, typ.String(), version, "pkg") + binDir = path.Join(am.dir, typ.String(), version, "bin") ) artifactFile, err := am.download(ctx, typ, version, pkgDir) @@ -290,6 +279,15 @@ func (am *ArtifactManager) getGreptimeLatestVersion() (string, error) { } func (am *ArtifactManager) greptimeDownloadURL(version string) (string, error) { + if version == "latest" { + // Get the latest greptime released version. + latestVersion, err := am.getGreptimeLatestVersion() + if err != nil { + return "", err + } + version = latestVersion + } + newVersion, err := am.isBreakingVersion(version) if err != nil { return "", err @@ -326,6 +324,15 @@ func (am *ArtifactManager) etcdDownloadURL(version string) (string, error) { } func (am *ArtifactManager) isBreakingVersion(version string) (bool, error) { + if version == "latest" { + // Get the latest greptime released version. + latestVersion, err := am.getGreptimeLatestVersion() + if err != nil { + return false, err + } + version = latestVersion + } + newVersion, err := semverutils.Compare(version, BreakingChangeVersion) if err != nil { return false, err diff --git a/pkg/deployer/baremetal/artifacts_test.go b/pkg/deployer/baremetal/artifacts_test.go index 1818c5f3..316ef36a 100644 --- a/pkg/deployer/baremetal/artifacts_test.go +++ b/pkg/deployer/baremetal/artifacts_test.go @@ -16,21 +16,48 @@ package baremetal import ( "context" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/logger" "os" - "sigs.k8s.io/kind/pkg/log" "testing" + + "sigs.k8s.io/kind/pkg/log" + + "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + "github.com/GreptimeTeam/gtctl/pkg/logger" +) + +const ( + testDir = "/tmp/gtctl-test-am" ) func TestArtifactManager(t *testing.T) { - am, err := NewArtifactManager("/tmp/gtctl-test-am", logger.New(os.Stdout, log.Level(4), logger.WithColored()), false) + am, err := NewArtifactManager(testDir, logger.New(os.Stdout, log.Level(4), logger.WithColored()), false) if err != nil { t.Errorf("failed to create artifact manager: %v", err) } + // Cleanup test directory. + defer func() { + os.RemoveAll(testDir) + }() + + testConfigs := []*config.Artifact{ + { + Version: "latest", + }, + { + Version: BreakingChangeVersion, + }, + } + ctx := context.Background() - if err := am.PrepareArtifact(ctx, GreptimeArtifactType, &config.Artifact{Version: "latest"}); err != nil { - t.Errorf("failed to prepare latest greptime artifact: %v", err) + for _, tc := range testConfigs { + if err := am.PrepareArtifact(ctx, GreptimeArtifactType, tc); err != nil { + t.Errorf("failed to prepare artifact: %v", err) + } + + _, err := am.BinaryPath(GreptimeArtifactType, tc) + if err != nil { + t.Errorf("failed to get binary path: %v", err) + } } }