From c3f774b5f0c66e9122d77960a2bfab9f62fe4097 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Tue, 5 Sep 2023 13:36:55 +0300 Subject: [PATCH 1/2] Enable lifecycle release bundle distribute --- documentation/CLI-for-JFrog-Lifecycle.md | 66 +++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- lifecycle/cli.go | 3 +- lifecycle_test.go | 33 +++++++++-- testdata/filespecs/search_all_dev_repo.json | 7 +++ utils/tests/consts.go | 32 ++++++---- 7 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 testdata/filespecs/search_all_dev_repo.json diff --git a/documentation/CLI-for-JFrog-Lifecycle.md b/documentation/CLI-for-JFrog-Lifecycle.md index 6a998e4c3..ce64fff87 100644 --- a/documentation/CLI-for-JFrog-Lifecycle.md +++ b/documentation/CLI-for-JFrog-Lifecycle.md @@ -121,3 +121,69 @@ Use signing key pair "myKeyPair" and overwrite in case of conflicts. ``` jf rbp --signing-key=myKeyPair --project=project0 --overwrite=true --sync=true myApp 1.0.0 PROD ``` + +### Distributing a release bundle + +This command distributes a Release Bundle v2 to an edge node. + +| | | +|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Command-name | release-bundle-distribute | +| Abbreviation | rbd | +| Command options | | +| --create-repo | \[Default: false\]

Set to true to create the repository on the edge if it does not exist. | +| --dry-run | \[Default: false\]

Set to true to disable communication with JFrog Distribution. | +| --dist-rules | \[Optional\]

Path to a file, which includes the Distribution Rules in a JSON format. See [structure](#distribution-rules-structure) bellow. | +| --site | \[Default: *\]

Wildcard filter for site name. | +| --city | \[Default: *\]

Wildcard filter for site city name. | +| --country-codes | \[Default: *\]

Semicolon-separated list of wildcard filters for site country codes. | +| --insecure-tls | \[Default: false\]

Set to true to skip TLS certificates verification. | +| --mapping-pattern | \[Optional\]

Specify along with 'mapping-target' to distribute artifacts to a different path on the edge node. You can use wildcards to specify multiple artifacts. | +| --mapping-target | \[Optional\]

The target path for distributed artifacts on the edge node. If not specified, the artifacts will have the same path and name on the edge node, as on the source Artifactory server. For flexibility in specifying the distribution path, you can include [placeholders](https://www.jfrog.com/confluence/display/CLI/CLI+for+JFrog+Artifactory#CLIforJFrogArtifactory-UsingPlaceholders) in the form of {1}, {2} which are replaced by corresponding tokens in the pattern path that are enclosed in parenthesis. | +| Command arguments | | +| release bundle name | Name of the Release Bundle to distribute. | +| release bundle version | Version of the Release Bundle to distribute. | + +#### Distribution Rules Structure: + ```json + { + "distribution_rules": [ + { + "site_name": "DC-1", + "city_name": "New-York", + "country_codes": ["1"] + }, + { + "site_name": "DC-2", + "city_name": "Tel-Aviv", + "country_codes": ["972"] + } + ] + } + ``` +The Distribution Rules format also supports wildcards. For example: + ```json + { + "distribution_rules": [ + { + "site_name": "", + "city_name": "", + "country_codes": ["*"] + } + ] + } + ``` + +#### Examples + +#### Example 1 +Distribute the Release Bundle v2 named myApp with version 1.0.0. Use the distribution rules defined in the specified file. + + jf rbd --dist-rules=/path/to/dist-rules.json myApp 1.0.0 + +#### Example 2 + +Distribute the Release Bundle v2 named myApp with version 1.0.0 using the default distribution rules. +Map files under the 'source' directory to be placed under the 'target' directory. + + jf rbd --dist-rules=/path/to/dist-rules.json --mapping-pattern="(*)/source/(*)" --mapping-target="{1}/target/{2}" myApp 1.0.0 \ No newline at end of file diff --git a/go.mod b/go.mod index 370930136..8dd70897b 100644 --- a/go.mod +++ b/go.mod @@ -129,6 +129,6 @@ replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27 -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230904070427-f3f2a103d9e2 +replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20230905103446-1b2c92667272 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc diff --git a/go.sum b/go.sum index c7b107794..570b639c9 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/RobiNino/jfrog-client-go v0.0.0-20230905103446-1b2c92667272 h1:pcTIXjWP0yxWiuoNHWQlaYoDleF/1Z3JD+F6m10PE7I= +github.com/RobiNino/jfrog-client-go v0.0.0-20230905103446-1b2c92667272/go.mod h1:uUnMrqHX7Xi+OCaZEE4b3BtsmGeOSCB7XqaEWVXEH/E= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -241,8 +243,6 @@ github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230904170814-03a7938bae94 h1:rAV/hUZNruzYOfGUqE5ubPY55NMtS59llfTcWGwofSg= github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230904170814-03a7938bae94/go.mod h1:eQJU+jDTcPVqOeHNheURqkMEMXDVNQFKKjcyVEa0058= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230904070427-f3f2a103d9e2 h1:uPDFvUv+3kAUyTR6gAhROAgsjnPe2jsxzqJSkjbIHmE= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230904070427-f3f2a103d9e2/go.mod h1:uUnMrqHX7Xi+OCaZEE4b3BtsmGeOSCB7XqaEWVXEH/E= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jszwec/csvutil v1.8.0 h1:G7vS2LGdpZZDH1HmHeNbxOaJ/ZnJlpwGFvOkTkJzzNk= diff --git a/lifecycle/cli.go b/lifecycle/cli.go index 647963c10..eaf201222 100644 --- a/lifecycle/cli.go +++ b/lifecycle/cli.go @@ -55,7 +55,6 @@ func GetCommands() []cli.Command { ArgsUsage: common.CreateEnvVars(), BashComplete: coreCommon.CreateBashCompletionFunc(), Category: lcCategory, - Hidden: true, Action: distribute, }, }) @@ -170,7 +169,7 @@ func assertSigningKeyProvided(c *cli.Context) error { } func createLifecycleDetailsByFlags(c *cli.Context) (*coreConfig.ServerDetails, error) { - lcDetails, err := cliutils.CreateServerDetailsWithConfigOffer(c, false, cliutils.Platform) + lcDetails, err := cliutils.CreateServerDetailsWithConfigOffer(c, true, cliutils.Platform) if err != nil { return nil, err } diff --git a/lifecycle_test.go b/lifecycle_test.go index 5a1516df8..16fc61248 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -14,12 +14,14 @@ import ( "github.com/jfrog/jfrog-client-go/lifecycle" "github.com/jfrog/jfrog-client-go/lifecycle/services" clientUtils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/distribution" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/stretchr/testify/assert" "net/http" "os" "path/filepath" "testing" + "time" ) const ( @@ -64,14 +66,17 @@ func TestLifecycle(t *testing.T) { promoteRb(t, lcManager, number3) // Verify the artifacts of both the initial release bundles made it to the prod repo. - searchSpec, err := tests.CreateSpec(tests.SearchAllProdRepo) + searchProdSpec, err := tests.CreateSpec(tests.SearchAllProdRepo) assert.NoError(t, err) - inttestutils.VerifyExistInArtifactory(tests.GetExpectedLifecycleArtifacts(), searchSpec, serverDetails, t) + inttestutils.VerifyExistInArtifactory(tests.GetExpectedLifecycleArtifacts(), searchProdSpec, serverDetails, t) distributeRb(t) + defer remoteDeleteReleaseBundle(t, lcManager, tests.LcRbName3, number3) + // Verify the artifacts were distributed correctly by the provided path mappings. - expected := append(tests.GetExpectedLifecycleArtifacts(), tests.GetExpectedLifecycleMappingArtifacts()...) - inttestutils.VerifyExistInArtifactory(expected, searchSpec, serverDetails, t) + searchDevSpec, err := tests.CreateSpec(tests.SearchAllDevRepo) + assert.NoError(t, err) + inttestutils.VerifyExistInArtifactory(tests.GetExpectedLifecycleDistributedArtifacts(), searchDevSpec, serverDetails, t) } func uploadBuilds(t *testing.T) func() { @@ -107,9 +112,11 @@ func distributeRb(t *testing.T) { assert.NoError(t, lcCli.Exec( "rbd", tests.LcRbName3, number3, getOption(cliutils.DistRules, distributionRulesPath), - getOption(cliutils.PathMappingPattern, tests.RtProdRepo+"/(*)"), - getOption(cliutils.PathMappingTarget, tests.RtProdRepo+"/target/{1}"), + getOption(cliutils.PathMappingPattern, "(*)/(*)"), + getOption(cliutils.PathMappingTarget, "{1}/target/{2}"), )) + // Wait after distribution before asserting. Can be removed once distribute supports sync. + time.Sleep(5 * time.Second) } func getOption(option, value string) string { @@ -187,6 +194,20 @@ func deleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServicesMan assert.NoError(t, lcManager.DeleteReleaseBundle(rbDetails, services.ReleaseBundleQueryParams{Async: false})) } +func remoteDeleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion string) { + params := distribution.NewDistributeReleaseBundleParams(rbName, rbVersion) + rules := &distribution.DistributionCommonParams{ + SiteName: "*", + CityName: "*", + CountryCodes: []string{"*"}, + } + params.DistributionRules = append(params.DistributionRules, rules) + + assert.NoError(t, lcManager.RemoteDeleteReleaseBundle(params, false)) + // Wait after remote deleting. Can be removed once remote deleting supports sync. + time.Sleep(5 * time.Second) +} + func uploadBuild(t *testing.T, specFileName, buildName, buildNumber string) { specFile, err := tests.CreateSpec(specFileName) assert.NoError(t, err) diff --git a/testdata/filespecs/search_all_dev_repo.json b/testdata/filespecs/search_all_dev_repo.json new file mode 100644 index 000000000..781226f0d --- /dev/null +++ b/testdata/filespecs/search_all_dev_repo.json @@ -0,0 +1,7 @@ +{ + "files": [ + { + "pattern": "${DEV_REPO}/*" + } + ] +} \ No newline at end of file diff --git a/utils/tests/consts.go b/utils/tests/consts.go index f415aa1f0..fdd8729c0 100644 --- a/utils/tests/consts.go +++ b/utils/tests/consts.go @@ -91,6 +91,7 @@ const ( SearchAllMaven = "search_all_maven.json" SearchAllNpm = "search_all_npm.json" SearchAllRepo1 = "search_all_repo1.json" + SearchAllDevRepo = "search_all_dev_repo.json" SearchAllProdRepo = "search_all_prod_repo.json" SearchDistRepoByInSuffix = "search_dist_repo_by_in_suffix.json" SearchRepo1ByInSuffix = "search_repo1_by_in_suffix.json" @@ -2099,17 +2100,26 @@ func GetExpectedLifecycleArtifacts() []string { } } -func GetExpectedLifecycleMappingArtifacts() []string { - return []string{ - RtProdRepo + "/target/a1.in", - RtProdRepo + "/target/a2.in", - RtProdRepo + "/target/a3.in", - RtProdRepo + "/target/b1.in", - RtProdRepo + "/target/b2.in", - RtProdRepo + "/target/b3.in", - RtProdRepo + "/target/c1.in", - RtProdRepo + "/target/c2.in", - RtProdRepo + "/target/c3.in", +func GetExpectedLifecycleDistributedArtifacts() []string { + return []string{ + RtDevRepo + "/a1.in", + RtDevRepo + "/a2.in", + RtDevRepo + "/a3.in", + RtDevRepo + "/b1.in", + RtDevRepo + "/b2.in", + RtDevRepo + "/b3.in", + RtDevRepo + "/c1.in", + RtDevRepo + "/c2.in", + RtDevRepo + "/c3.in", + RtDevRepo + "/target/a1.in", + RtDevRepo + "/target/a2.in", + RtDevRepo + "/target/a3.in", + RtDevRepo + "/target/b1.in", + RtDevRepo + "/target/b2.in", + RtDevRepo + "/target/b3.in", + RtDevRepo + "/target/c1.in", + RtDevRepo + "/target/c2.in", + RtDevRepo + "/target/c3.in", } } From db422fabb23bda73a1b5eadd9f90079871317000 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 13 Sep 2023 19:49:51 +0300 Subject: [PATCH 2/2] update go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0fd0dc5f8..9fa03905f 100644 --- a/go.mod +++ b/go.mod @@ -130,6 +130,6 @@ require ( // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27 -replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20230913162257-3b17f88a2266 +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230913164821-c396cb9a4e06 // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc diff --git a/go.sum b/go.sum index a61f2aaad..b1a0f185a 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,6 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/RobiNino/jfrog-client-go v0.0.0-20230913162257-3b17f88a2266 h1:3vsUW82FMHMWlRPnXmSGPQyivSvnTLtDgugmkqTJb98= -github.com/RobiNino/jfrog-client-go v0.0.0-20230913162257-3b17f88a2266/go.mod h1:UewnwkIf/77HzBgwCPzOHZCK6V/Nw5/JwdzN/tRb4aU= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -243,6 +241,8 @@ github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-cli-core/v2 v2.43.0 h1:euo1CjZcpMdWkFUQ3zffRPfCR1zXhLD6TE/lfexV99o= github.com/jfrog/jfrog-cli-core/v2 v2.43.0/go.mod h1:NWqT0ZnAvEdjaXGp64POvRV35TJ2R/c0W45UmrXQonk= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230913164821-c396cb9a4e06 h1:QcREjpuUJmLsy7aSXTcMomAIVFgOsdUHGi7lcqVfg8M= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230913164821-c396cb9a4e06/go.mod h1:UewnwkIf/77HzBgwCPzOHZCK6V/Nw5/JwdzN/tRb4aU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jszwec/csvutil v1.8.0 h1:G7vS2LGdpZZDH1HmHeNbxOaJ/ZnJlpwGFvOkTkJzzNk=