Skip to content

Commit

Permalink
modules: extract backupspec to the submodule (#4226)
Browse files Browse the repository at this point in the history
  • Loading branch information
karol-kokoszka authored Feb 6, 2025
1 parent ef0b215 commit c555047
Show file tree
Hide file tree
Showing 70 changed files with 1,366 additions and 468 deletions.
21 changes: 21 additions & 0 deletions backupspec/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
module github.com/scylladb/scylla-manager/backupspec

go 1.23.2

require (
github.com/gocql/gocql v1.6.0
github.com/google/go-cmp v0.6.0
github.com/json-iterator/go v1.1.12
github.com/pkg/errors v0.9.1
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168
go.uber.org/multierr v1.11.0
)

require (
github.com/gobwas/glob v0.2.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
)
46 changes: 46 additions & 0 deletions backupspec/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU=
github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168 h1:ua8SUnAgg1soc8zP5TInWfNpoWilBah8Mavdll7TxTs=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168/go.mod h1:VKqHSrDj9zfgCKilpbdpmqV1TjmSglt/df79eIg6wuI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
// Copyright (C) 2017 ScyllaDB

package backupspec

import (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@ import (

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
. "github.com/scylladb/scylla-manager/v3/pkg/testutils"
"github.com/scylladb/scylla-manager/v3/pkg/util/uuid"
)

func TestManifestInfoParsePath(t *testing.T) {
t.Parallel()

opts := cmp.Options{
UUIDComparer(),
cmp.Comparer(func(a, b uuid.UUID) bool { return a == b }),
cmpopts.IgnoreUnexported(ManifestInfo{}),
}

Expand Down
File renamed without changes.
9 changes: 5 additions & 4 deletions pkg/service/backup/backupspec/tags.go → backupspec/tags.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ import (
)

var (
tagDateFormat = "20060102150405"
tagRegexp = regexp.MustCompile("^sm_([0-9]{14})UTC$")
errInvalidSnapshotTag = errors.New("not a Scylla Manager snapshot tag, expected format is sm_20060102150405UTC")
ErrInvalidSnapshotTag = errors.New("not a Scylla Manager snapshot tag, expected format is sm_20060102150405UTC")

tagDateFormat = "20060102150405"
tagRegexp = regexp.MustCompile("^sm_([0-9]{14})UTC$")
)

// NewSnapshotTag creates new snapshot tag for the current time.
Expand All @@ -35,7 +36,7 @@ func IsSnapshotTag(tag string) bool {
func SnapshotTagTime(tag string) (time.Time, error) {
m := tagRegexp.FindStringSubmatch(tag)
if m == nil {
return time.Time{}, errInvalidSnapshotTag
return time.Time{}, ErrInvalidSnapshotTag
}
return timeutc.Parse(tagDateFormat, m[1])
}
File renamed without changes.
1 change: 0 additions & 1 deletion cloud/backup-cleaner/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"time"

"github.com/scylladb/go-set/strset"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/util/timeutc"
)

Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ require (
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/hashicorp/go-version v1.7.0
github.com/json-iterator/go v1.1.12
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.20.0
Expand All @@ -31,8 +30,9 @@ require (
github.com/scylladb/go-reflectx v1.0.1
github.com/scylladb/go-set v1.0.2
github.com/scylladb/gocqlx/v2 v2.8.0
github.com/scylladb/scylla-manager/backupspec v1.0.1
github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20250122142320-e1127475cc4c
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250122142320-e1127475cc4c
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168
github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250122142320-e1127475cc4c
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5
Expand Down Expand Up @@ -83,6 +83,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/lnquy/cron v1.1.1 // indirect
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1062,10 +1062,12 @@ github.com/scylladb/google-api-go-client v0.34.1-patched h1:DW+T0HA+74o6FDr3TFzV
github.com/scylladb/google-api-go-client v0.34.1-patched/go.mod h1:RriRmS2wJXH+2yd9PRTEcR380U9AXmurWwznqVhzsSc=
github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e h1:lJRphCtu+nKd+mfo8whOTeFkgjMWvk8iCSlqgibKSa8=
github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e/go.mod h1:JGZp4EvCUK+6AM1Fe1dye5xvihTc/Bk0WnHHSCJOePM=
github.com/scylladb/scylla-manager/backupspec v1.0.1 h1:WaFVHEWwVA3WUeCf2WakmhQ59o+wTYdzo4+qQf8maXk=
github.com/scylladb/scylla-manager/backupspec v1.0.1/go.mod h1:izqe7mR8wpSFUTjwM3ptUHIk3nF+F5Bipmm0FPRqnlI=
github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20250122142320-e1127475cc4c h1:scmfi5j0Sgl7i3g6tVsu5yhCBlnT/PLN/PYVaY8B+Hc=
github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20250122142320-e1127475cc4c/go.mod h1:7+yHDmds+qO/8b0w4xTptCLh/Gr/hXyAjIIZkMj4KHk=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250122142320-e1127475cc4c h1:RLgSH0r/TI+Aw1f2bHUqnjVOI+8cOc9eX5Kr73N9Iuc=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250122142320-e1127475cc4c/go.mod h1:VKqHSrDj9zfgCKilpbdpmqV1TjmSglt/df79eIg6wuI=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168 h1:ua8SUnAgg1soc8zP5TInWfNpoWilBah8Mavdll7TxTs=
github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20250123113205-a6303e03d168/go.mod h1:VKqHSrDj9zfgCKilpbdpmqV1TjmSglt/df79eIg6wuI=
github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250122142320-e1127475cc4c h1:1qkWdf5FbOwW3iE712s7mwOFSHaTRf9vlZ1LW+HDmVE=
github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250122142320-e1127475cc4c/go.mod h1:nCN5P0jiWL0W7jbcZ9p0ndtZAPoyEWXefddx/nbyFes=
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4 h1:8qmTC5ByIXO3GP/IzBkxcZ/99VITvnIETDhdFz/om7A=
Expand Down
6 changes: 4 additions & 2 deletions pkg/cmd/agent/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,19 @@ import (

"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/scylladb/scylla-manager/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/rclone"
"github.com/scylladb/scylla-manager/v3/pkg/rclone/bench"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup"

"github.com/scylladb/scylla-manager/v3/pkg/util/timeutc"
"github.com/spf13/cobra"
"go.uber.org/zap/zapcore"
)

var benchmarkArgs = struct {
dirGlob []string
location backupspec.LocationValue
location backup.LocationValue

configFiles []string
debug bool
Expand Down
6 changes: 4 additions & 2 deletions pkg/cmd/agent/check_location.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,17 @@ import (

"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/scylladb/scylla-manager/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/rclone/operations"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup"

"github.com/spf13/cobra"
"go.uber.org/zap/zapcore"
)

var checkLocationArgs = struct {
configFiles []string
location backupspec.LocationValue
location backup.LocationValue
debug bool
}{}

Expand Down
6 changes: 4 additions & 2 deletions pkg/cmd/agent/download_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@ import (
api "github.com/go-openapi/runtime/client"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
"github.com/scylladb/scylla-manager/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/downloader"
"github.com/scylladb/scylla-manager/v3/pkg/rclone"
backup "github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/service/backup"

"github.com/scylladb/scylla-manager/v3/pkg/util/uuid"
scyllaOperations "github.com/scylladb/scylla-manager/v3/swagger/gen/scylla/v1/client/operations"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -206,7 +208,7 @@ func init() {
a := &downloadFilesArgs

f.StringSliceVarP(&a.configFiles, "config-file", "c", []string{"/etc/scylla-manager-agent/scylla-manager-agent.yaml"}, "configuration file `path`")
f.VarP(&a.location, "location", "L", "backup location in the format <provider>:<name> e.g. s3:my-bucket, the supported providers are: "+strings.Join(backup.Providers(), ", ")) //nolint: lll
f.VarP(&a.location, "location", "L", "backup location in the format <provider>:<name> e.g. s3:my-bucket, the supported providers are: "+strings.Join(backupspec.Providers(), ", ")) //nolint: lll
f.StringVarP(&a.dataDir, "data-dir", "d", "", "`path` to Scylla data directory (typically /var/lib/scylla/data) or other directory to use for downloading the files (default current directory)") //nolint: lll
f.StringSliceVarP(&a.keyspace, "keyspace", "K", nil, "a comma-separated `list` of keyspace/tables glob patterns, e.g. 'keyspace,!keyspace.table_prefix_*'")
f.Var(&a.mode, "mode", "mode changes resulting directory structure, supported values are: `upload, sstableloader`, set 'upload' to use table upload directories, set 'sstableloader' for <keyspace>/<table> directories layout") // nolint: lll
Expand Down
22 changes: 11 additions & 11 deletions pkg/downloader/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"github.com/scylladb/go-log"
backup "github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/util/inexlist/ksfilter"
"github.com/scylladb/scylla-manager/v3/pkg/util/parallel"
)
Expand All @@ -37,7 +37,7 @@ type Downloader struct {
fdst fs.Fs
}

func New(l backup.Location, dataDir string, logger log.Logger, opts ...Option) (*Downloader, error) {
func New(l backupspec.Location, dataDir string, logger log.Logger, opts ...Option) (*Downloader, error) {
// Temporary context to satisfy rclone.
ctx := context.Background()

Expand Down Expand Up @@ -70,20 +70,20 @@ func New(l backup.Location, dataDir string, logger log.Logger, opts ...Option) (
}

// DryRun returns an action plan without performing any disk operations.
func (d *Downloader) DryRun(ctx context.Context, m backup.ManifestInfoWithContent) (Plan, error) {
func (d *Downloader) DryRun(ctx context.Context, m backupspec.ManifestInfoWithContent) (Plan, error) {
d.dryRun = true
d.plan = Plan{m: m}
return d.plan, d.download(ctx, m, 1)
}

// Download executes download operation by taking snapshot files from configured
// locations and downloading them to the data directory.
func (d *Downloader) Download(ctx context.Context, m backup.ManifestInfoWithContent) error {
func (d *Downloader) Download(ctx context.Context, m backupspec.ManifestInfoWithContent) error {
d.dryRun = false
return d.download(ctx, m, parallel.NoLimit)
}

func (d *Downloader) download(ctx context.Context, m backup.ManifestInfoWithContent, workers int) error {
func (d *Downloader) download(ctx context.Context, m backupspec.ManifestInfoWithContent, workers int) error {
d.logger.Info(ctx, "Initializing downloader",
"cluster_id", m.ClusterID,
"cluster_name", m.ClusterName,
Expand Down Expand Up @@ -182,8 +182,8 @@ func (d *Downloader) download(ctx context.Context, m backup.ManifestInfoWithCont
return parallel.Run(len(index), workers, f, notify)
}

func (d *Downloader) filteredIndex(ctx context.Context, m backup.ManifestInfoWithContent) ([]backup.FilesMeta, error) {
var i []backup.FilesMeta
func (d *Downloader) filteredIndex(ctx context.Context, m backupspec.ManifestInfoWithContent) ([]backupspec.FilesMeta, error) {
var i []backupspec.FilesMeta

if m.Index != nil {
i = m.Index
Expand All @@ -199,7 +199,7 @@ func (d *Downloader) filteredIndex(ctx context.Context, m backup.ManifestInfoWit
return i, nil
}

var index []backup.FilesMeta
var index []backupspec.FilesMeta
for _, u := range i {
if !d.shouldDownload(u.Keyspace, u.Table) {
d.logger.Debug(ctx, "Table filtered out", "keyspace", u.Keyspace, "table", u.Table)
Expand All @@ -214,7 +214,7 @@ func (d *Downloader) shouldDownload(keyspace, table string) bool {
return d.keyspace == nil || d.keyspace.Check(keyspace, table)
}

func (d *Downloader) clearTableIfNeeded(ctx context.Context, u backup.FilesMeta) error {
func (d *Downloader) clearTableIfNeeded(ctx context.Context, u backupspec.FilesMeta) error {
if !d.clearTables {
return nil
}
Expand Down Expand Up @@ -274,7 +274,7 @@ func (d *Downloader) clearTableIfNeeded(ctx context.Context, u backup.FilesMeta)
return nil
}

func (d *Downloader) downloadFiles(ctx context.Context, m backup.ManifestInfoWithContent, u backup.FilesMeta) error {
func (d *Downloader) downloadFiles(ctx context.Context, m backupspec.ManifestInfoWithContent, u backupspec.FilesMeta) error {
d.logger.Info(ctx, "Downloading",
"keyspace", u.Keyspace,
"table", u.Table,
Expand All @@ -295,7 +295,7 @@ func (d *Downloader) downloadFiles(ctx context.Context, m backup.ManifestInfoWit
return sync.CopyPaths(ctx, d.fdst, d.dstDir(u), d.fsrc, m.SSTableVersionDir(u.Keyspace, u.Table, u.Version), u.Files, false)
}

func (d *Downloader) dstDir(u backup.FilesMeta) (dir string) {
func (d *Downloader) dstDir(u backupspec.FilesMeta) (dir string) {
switch d.mode {
case DefaultTableDirMode:
dir = path.Join(u.Keyspace, u.Table+"-"+u.Version)
Expand Down
19 changes: 10 additions & 9 deletions pkg/downloader/downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,17 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/scylladb/go-log"
"github.com/scylladb/scylla-manager/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/downloader"
backup "github.com/scylladb/scylla-manager/v3/pkg/service/backup/backupspec"
"github.com/scylladb/scylla-manager/v3/pkg/testutils"

"github.com/scylladb/scylla-manager/v3/pkg/util/uuid"
"go.uber.org/zap/zapcore"
)

func TestDownload(t *testing.T) {
var (
location = backup.Location{Provider: "testdata"}
location = backupspec.Location{Provider: "testdata"}
criteria = downloader.ManifestLookupCriteria{
NodeID: uuid.MustParse("942ba1b6-30a3-441e-ac3c-158864d8b861"),
SnapshotTag: "sm_20210215151954UTC",
Expand Down Expand Up @@ -184,7 +185,7 @@ func TestDownload(t *testing.T) {
}

func TestDownloadOwnerCheck(t *testing.T) {
location := backup.Location{Provider: "testdata"}
location := backupspec.Location{Provider: "testdata"}
d, err := downloader.New(location, "/", log.NewDevelopment())

err = d.Download(context.Background(), dummyManifest())
Expand All @@ -194,7 +195,7 @@ func TestDownloadOwnerCheck(t *testing.T) {
}

func TestDownloadFilterCheck(t *testing.T) {
location := backup.Location{Provider: "testdata"}
location := backupspec.Location{Provider: "testdata"}
d, err := downloader.New(location, ".", log.NewDevelopment(), downloader.WithKeyspace([]string{"bla"}))

err = d.Download(context.Background(), dummyManifest())
Expand All @@ -203,11 +204,11 @@ func TestDownloadFilterCheck(t *testing.T) {
}
}

func dummyManifest() backup.ManifestInfoWithContent {
return backup.ManifestInfoWithContent{
ManifestInfo: &backup.ManifestInfo{},
ManifestContentWithIndex: &backup.ManifestContentWithIndex{
Index: []backup.FilesMeta{
func dummyManifest() backupspec.ManifestInfoWithContent {
return backupspec.ManifestInfoWithContent{
ManifestInfo: &backupspec.ManifestInfo{},
ManifestContentWithIndex: &backupspec.ManifestContentWithIndex{
Index: []backupspec.FilesMeta{
{
Keyspace: "foo",
Table: "bar",
Expand Down
Loading

0 comments on commit c555047

Please sign in to comment.