From 0012f89e98fc292bc8a2752242903a332896f1bc Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 7 Feb 2023 22:28:13 +0100 Subject: [PATCH] fix(s3config): Accept S3 URL as url escaped. Fixes: https://github.com/grafana/loki/issues/1607 Introduced this type in another PR https://github.com/grafana/dskit/pull/265 --- go.mod | 2 +- go.sum | 2 + .../chunk/client/aws/s3_storage_client.go | 2 +- .../grafana/dskit/flagext/urlescaped.go | 63 +++++++++++++++++++ .../grafana/dskit/kv/consul/client.go | 2 +- .../grafana/dskit/modules/modules.go | 30 ++++++++- vendor/modules.txt | 2 +- 7 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 vendor/github.com/grafana/dskit/flagext/urlescaped.go diff --git a/go.mod b/go.mod index 18e6a86305747..4aee0a7f292f1 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 - github.com/grafana/dskit v0.0.0-20230201083518-528d8a7d52f2 + github.com/grafana/dskit v0.0.0-20230207212244-6b8c28bc192e github.com/grafana/go-gelf/v2 v2.0.1 github.com/grafana/gomemcache v0.0.0-20230105173749-11f792309e1f github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd diff --git a/go.sum b/go.sum index bcc9f1335b076..70f397a676c24 100644 --- a/go.sum +++ b/go.sum @@ -742,6 +742,8 @@ github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 h1:qhugDMdQ4 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2/go.mod h1:w/aiO1POVIeXUQyl0VQSZjl5OAGDTL5aX+4v0RA1tcw= github.com/grafana/dskit v0.0.0-20230201083518-528d8a7d52f2 h1:IOks+FXJ6iO/pfbaVEf4efNw+YzYBYNCkCabyrbkFTM= github.com/grafana/dskit v0.0.0-20230201083518-528d8a7d52f2/go.mod h1:zj+5BNZAVmQafV583uLTAOzRr963KPdEm4d6NPmtbwg= +github.com/grafana/dskit v0.0.0-20230207212244-6b8c28bc192e h1:TTyqiatfJ0z/uUZQzpQoymc0pz5ypBnJiGY0fHXkFgw= +github.com/grafana/dskit v0.0.0-20230207212244-6b8c28bc192e/go.mod h1:ulYLLoSd71AWIjxgifLO86Lndx82Yj+IcV+fFnh8tkI= github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak= github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90= github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 h1:xLuzPoOzdfNb/RF/IENCw+oLVdZB4G21VPhkHBgwSHY= diff --git a/pkg/storage/chunk/client/aws/s3_storage_client.go b/pkg/storage/chunk/client/aws/s3_storage_client.go index 7b23cecbb7df6..a7b138924f090 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client.go @@ -63,7 +63,7 @@ func init() { // S3Config specifies config for storing chunks on AWS S3. type S3Config struct { - S3 flagext.URLValue + S3 flagext.URLEscaped S3ForcePathStyle bool BucketNames string diff --git a/vendor/github.com/grafana/dskit/flagext/urlescaped.go b/vendor/github.com/grafana/dskit/flagext/urlescaped.go new file mode 100644 index 0000000000000..0bac0eb821bc5 --- /dev/null +++ b/vendor/github.com/grafana/dskit/flagext/urlescaped.go @@ -0,0 +1,63 @@ +package flagext + +import "net/url" + +// URLEscaped is a url.URL that can be used as a flag. +// URL value it contains will always be URL escaped and safe. +type URLEscaped struct { + *url.URL +} + +// String implements flag.Value +func (v URLEscaped) String() string { + if v.URL == nil { + return "" + } + return v.URL.String() +} + +// Set implements flag.Value +// Set make sure given URL string is escaped. +func (v *URLEscaped) Set(s string) error { + s = url.QueryEscape(s) + + u, err := url.Parse(s) + if err != nil { + return err + } + v.URL = u + return nil +} + +// UnmarshalYAML implements yaml.Unmarshaler. +func (v *URLEscaped) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + + // An empty string means no URL has been configured. + if s == "" { + v.URL = nil + return nil + } + + return v.Set(s) +} + +// Marshalyaml Implements yaml.Marshaler. +func (v URLEscaped) MarshalYAML() (interface{}, error) { + if v.URL == nil { + return "", nil + } + + // Mask out passwords when marshalling URLs back to YAML. + u := *v.URL + if u.User != nil { + if _, set := u.User.Password(); set { + u.User = url.UserPassword(u.User.Username(), "********") + } + } + + return u.String(), nil +} diff --git a/vendor/github.com/grafana/dskit/kv/consul/client.go b/vendor/github.com/grafana/dskit/kv/consul/client.go index c7ef8d372e709..89d7297fbcd43 100644 --- a/vendor/github.com/grafana/dskit/kv/consul/client.go +++ b/vendor/github.com/grafana/dskit/kv/consul/client.go @@ -252,7 +252,7 @@ func (c *Client) WatchKey(ctx context.Context, key string, f func(interface{}) b } if kvp == nil { - level.Info(c.logger).Log("msg", "value is nil", "key", key, "index", index) + level.Debug(c.logger).Log("msg", "value is nil", "key", key, "index", index) continue } diff --git a/vendor/github.com/grafana/dskit/modules/modules.go b/vendor/github.com/grafana/dskit/modules/modules.go index 0a28797489bea..46734e2dee785 100644 --- a/vendor/github.com/grafana/dskit/modules/modules.go +++ b/vendor/github.com/grafana/dskit/modules/modules.go @@ -18,8 +18,11 @@ type module struct { // initFn for this module (can return nil) initFn func() (services.Service, error) - // is this module user visible (i.e. intended to be passed to `InitModuleServices`) + // is this module user visible userVisible bool + + // is the module allowed to be selected as a target + targetable bool } // Manager is a component that initialises modules of the application @@ -32,6 +35,18 @@ type Manager struct { // UserInvisibleModule is an option for `RegisterModule` that marks module not visible to user. Modules are user visible by default. func UserInvisibleModule(m *module) { m.userVisible = false + + // by default invisible modules should not be targetable. + m.targetable = false +} + +// UserInvisibleTargetableModule is an option for `RegisterModule` that marks module not visible to user, but still keeps it targetable. +// Modules are visible and targetable by default. +func UserInvisibleTargetableModule(m *module) { + m.userVisible = false + + // ensure that the module is still targetable. + m.targetable = true } // NewManager creates a new Manager @@ -49,6 +64,7 @@ func (m *Manager) RegisterModule(name string, initFn func() (services.Service, e m.modules[name] = &module{ initFn: initFn, userVisible: true, + targetable: true, } for _, o := range options { @@ -166,6 +182,18 @@ func (m *Manager) IsUserVisibleModule(mod string) bool { return false } +// IsTargetableModule check if given module is targetable or not. Returns true +// if and only if the given module is registered and is targetable. +func (m *Manager) IsTargetableModule(mod string) bool { + val, ok := m.modules[mod] + + if ok { + return val.targetable + } + + return false +} + // IsModuleRegistered checks if the given module has been registered or not. Returns true // if the module has previously been registered via a call to RegisterModule, false otherwise. func (m *Manager) IsModuleRegistered(mod string) bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index 66b30aafa56ac..656fbba1b0bc8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -681,7 +681,7 @@ github.com/gorilla/websocket # github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 ## explicit; go 1.17 github.com/grafana/cloudflare-go -# github.com/grafana/dskit v0.0.0-20230201083518-528d8a7d52f2 +# github.com/grafana/dskit v0.0.0-20230207212244-6b8c28bc192e ## explicit; go 1.18 github.com/grafana/dskit/backoff github.com/grafana/dskit/concurrency