Skip to content

Commit 336ade4

Browse files
feat: support for credentials in AWS Secrets Manager (#174)
* feat: support for credentials in AWS Secrets Manager * new field for secretsmanager instead of hacking existing password field * improve docs * fix tests * unit test fetching credentials * add flag for specifying secrets manager arn and improve error message * more documentation
1 parent 484da8b commit 336ade4

File tree

17 files changed

+258
-72
lines changed

17 files changed

+258
-72
lines changed

README.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,26 @@ using [`os.ExpandEnv`](https://pkg.go.dev/os#ExpandEnv) at load time. The latter
338338
references of the form `$ENV_VAR_NAME` or `${ENV_VAR_NAME}` with the associated values from the
339339
environment.
340340

341+
Additionally, the Amazon Resource Name (ARN) of a secret in AWS Secrets Manager can be provided
342+
instead of the username and password. Topicctl will then retrieve the secret value from Secrets
343+
Manager and use it as the credentials. The secret in Secrets Manager must have a value in the format
344+
shown below, identical to what [AWS MSK requires](https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html#msk-password-tutorial).
345+
```json
346+
{
347+
"username": "alice",
348+
"password": "alice-secret"
349+
}
350+
```
351+
352+
An example of secrets manager being used can be seen below. Be sure to include the [6Random-Characters
353+
AWS Secrets Manager tacks on to the end of a secrets ARN](https://docs.aws.amazon.com/secretsmanager/latest/userguide/getting-started.html).
354+
```yaml
355+
sasl:
356+
enabled: true
357+
mechanism: SCRAM-SHA-512
358+
secretsManagerArn: arn:aws:secretsmanager:<Region>:<AccountId>:secret:SecretName-6RandomCharacters
359+
```
360+
341361
### Topics
342362

343363
Each topic is configured in a YAML file. The following is an

cmd/topicctl/subcmd/apply.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,9 +197,12 @@ func applyTopic(
197197
adminClient, err = clusterConfig.NewAdminClient(
198198
ctx,
199199
nil,
200-
applyConfig.dryRun,
201-
applyConfig.shared.saslUsername,
202-
applyConfig.shared.saslPassword,
200+
config.AdminClientOpts{
201+
ReadOnly: applyConfig.dryRun,
202+
UsernameOverride: applyConfig.shared.saslUsername,
203+
PasswordOverride: applyConfig.shared.saslPassword,
204+
SecretsManagerArnOverride: applyConfig.shared.saslSecretsManagerArn,
205+
},
203206
)
204207
if err != nil {
205208
return err

cmd/topicctl/subcmd/bootstrap.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,12 @@ func bootstrapRun(cmd *cobra.Command, args []string) error {
7272
adminClient, err := clusterConfig.NewAdminClient(
7373
ctx,
7474
nil,
75-
true,
76-
bootstrapConfig.shared.saslUsername,
77-
bootstrapConfig.shared.saslPassword,
75+
config.AdminClientOpts{
76+
ReadOnly: true,
77+
UsernameOverride: bootstrapConfig.shared.saslUsername,
78+
PasswordOverride: bootstrapConfig.shared.saslPassword,
79+
SecretsManagerArnOverride: bootstrapConfig.shared.saslSecretsManagerArn,
80+
},
7881
)
7982
if err != nil {
8083
return err

cmd/topicctl/subcmd/check.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,12 @@ func checkTopicFile(
137137
adminClient, err = clusterConfig.NewAdminClient(
138138
ctx,
139139
nil,
140-
true,
141-
checkConfig.shared.saslUsername,
142-
checkConfig.shared.saslPassword,
140+
config.AdminClientOpts{
141+
ReadOnly: true,
142+
UsernameOverride: checkConfig.shared.saslUsername,
143+
PasswordOverride: checkConfig.shared.saslPassword,
144+
SecretsManagerArnOverride: checkConfig.shared.saslSecretsManagerArn,
145+
},
143146
)
144147
if err != nil {
145148
return false, err

cmd/topicctl/subcmd/create.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,12 @@ func createACL(
150150
adminClient, err = clusterConfig.NewAdminClient(
151151
ctx,
152152
nil,
153-
createConfig.dryRun,
154-
createConfig.shared.saslUsername,
155-
createConfig.shared.saslPassword,
153+
config.AdminClientOpts{
154+
ReadOnly: createConfig.dryRun,
155+
UsernameOverride: createConfig.shared.saslUsername,
156+
PasswordOverride: createConfig.shared.saslPassword,
157+
SecretsManagerArnOverride: createConfig.shared.saslSecretsManagerArn,
158+
},
156159
)
157160
if err != nil {
158161
return err

cmd/topicctl/subcmd/rebalance.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,12 @@ func rebalanceRun(cmd *cobra.Command, args []string) error {
125125

126126
adminClient, err := clusterConfig.NewAdminClient(ctx,
127127
nil,
128-
rebalanceConfig.dryRun,
129-
rebalanceConfig.shared.saslUsername,
130-
rebalanceConfig.shared.saslPassword,
128+
config.AdminClientOpts{
129+
ReadOnly: rebalanceConfig.dryRun,
130+
UsernameOverride: rebalanceConfig.shared.saslUsername,
131+
PasswordOverride: rebalanceConfig.shared.saslPassword,
132+
SecretsManagerArnOverride: rebalanceConfig.shared.saslSecretsManagerArn,
133+
},
131134
)
132135
if err != nil {
133136
log.Fatal(err)

cmd/topicctl/subcmd/shared.go

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,21 @@ import (
1414
)
1515

1616
type sharedOptions struct {
17-
brokerAddr string
18-
clusterConfig string
19-
expandEnv bool
20-
saslMechanism string
21-
saslPassword string
22-
saslUsername string
23-
tlsCACert string
24-
tlsCert string
25-
tlsEnabled bool
26-
tlsKey string
27-
tlsSkipVerify bool
28-
tlsServerName string
29-
zkAddr string
30-
zkPrefix string
17+
brokerAddr string
18+
clusterConfig string
19+
expandEnv bool
20+
saslMechanism string
21+
saslPassword string
22+
saslUsername string
23+
saslSecretsManagerArn string
24+
tlsCACert string
25+
tlsCert string
26+
tlsEnabled bool
27+
tlsKey string
28+
tlsSkipVerify bool
29+
tlsServerName string
30+
zkAddr string
31+
zkPrefix string
3132
}
3233

3334
func (s sharedOptions) validate() error {
@@ -76,7 +77,7 @@ func (s sharedOptions) validate() error {
7677
}
7778

7879
useTLS := s.tlsEnabled || s.tlsCACert != "" || s.tlsCert != "" || s.tlsKey != ""
79-
useSASL := s.saslMechanism != "" || s.saslPassword != "" || s.saslUsername != ""
80+
useSASL := s.saslMechanism != "" || s.saslPassword != "" || s.saslUsername != "" || s.saslSecretsManagerArn != ""
8081

8182
if useTLS && s.zkAddr != "" {
8283
log.Warn("TLS flags are ignored accessing cluster via zookeeper")
@@ -95,6 +96,10 @@ func (s sharedOptions) validate() error {
9596
(s.saslUsername != "" || s.saslPassword != "") {
9697
log.Warn("Username and password are ignored if using SASL AWS-MSK-IAM")
9798
}
99+
100+
if s.saslUsername != "" || s.saslPassword != "" && s.saslSecretsManagerArn != "" {
101+
err = multierror.Append(err, errors.New("Cannot set both sasl-username or sasl-password and sasl-secrets-manager-arn"))
102+
}
98103
}
99104

100105
return err
@@ -113,9 +118,12 @@ func (s sharedOptions) getAdminClient(
113118
return clusterConfig.NewAdminClient(
114119
ctx,
115120
sess,
116-
readOnly,
117-
s.saslUsername,
118-
s.saslPassword,
121+
config.AdminClientOpts{
122+
ReadOnly: readOnly,
123+
UsernameOverride: s.saslUsername,
124+
PasswordOverride: s.saslPassword,
125+
SecretsManagerArnOverride: s.saslSecretsManagerArn,
126+
},
119127
)
120128
} else if s.brokerAddr != "" {
121129
tlsEnabled := (s.tlsEnabled ||
@@ -150,10 +158,11 @@ func (s sharedOptions) getAdminClient(
150158
SkipVerify: s.tlsSkipVerify,
151159
},
152160
SASL: admin.SASLConfig{
153-
Enabled: saslEnabled,
154-
Mechanism: saslMechanism,
155-
Password: s.saslPassword,
156-
Username: s.saslUsername,
161+
Enabled: saslEnabled,
162+
Mechanism: saslMechanism,
163+
Password: s.saslPassword,
164+
Username: s.saslUsername,
165+
SecretsManagerArn: s.saslSecretsManagerArn,
157166
},
158167
},
159168
ReadOnly: readOnly,
@@ -211,6 +220,12 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) {
211220
os.Getenv("TOPICCTL_SASL_USERNAME"),
212221
"SASL username if using SASL; will override value set in cluster config",
213222
)
223+
cmd.PersistentFlags().StringVar(
224+
&options.saslSecretsManagerArn,
225+
"sasl-secrets-manager-arn",
226+
os.Getenv("TOPICCTL_SASL_SECRETS_MANAGER_ARN"),
227+
"Secrets Manager ARN to use for credentials if using SASL; will override value set in cluster config",
228+
)
214229
cmd.PersistentFlags().StringVar(
215230
&options.tlsCACert,
216231
"tls-ca-cert",

examples/auth/cluster.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,14 @@ spec:
3131
#
3232
username: adminscram
3333
password: admin-secret-512
34+
#
35+
# Another alternative is to omit these values and use secretsManagerArn to reference
36+
# an AWS Secrets Manager secret containing the username and password. More information
37+
# can be found in the README. An example can be seen below.
38+
# secretsManagerArn: arn:aws:secretsmanager:us-west-2:1000000000:secret:AmazonMSK_kafka-admin-wEiwjV
39+
#
40+
# This can also be set via:
41+
#
42+
# 1. The --sasl-secrets-manager-arn command-line flag,
43+
# 2. The TOPICCTL_SASL_SECRETS_MANAGER_ARN environment variable, or
44+
# 3. Putting a placeholder string in the config and running with the --expand-env flag as

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/segmentio/topicctl
33
go 1.18
44

55
require (
6-
github.com/aws/aws-sdk-go v1.44.208
6+
github.com/aws/aws-sdk-go v1.49.12
77
github.com/briandowns/spinner v1.19.0
88
github.com/c-bata/go-prompt v0.2.3
99
github.com/fatih/color v1.13.0

go.sum

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
github.com/aws/aws-sdk-go v1.41.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
2-
github.com/aws/aws-sdk-go v1.44.208 h1:xk1E2zWAIskrOP+huXuCYFR9ZdQWfTVid8Cjiwj2H1o=
3-
github.com/aws/aws-sdk-go v1.44.208/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
2+
github.com/aws/aws-sdk-go v1.49.12 h1:SbGHDdMjtuTL8zpRXKjvIvQHLt9cCqcxcHoJps23WxI=
3+
github.com/aws/aws-sdk-go v1.49.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
44
github.com/briandowns/spinner v1.19.0 h1:s8aq38H+Qju89yhp89b4iIiMzMm8YN3p6vGpwyh/a8E=
55
github.com/briandowns/spinner v1.19.0/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU=
66
github.com/c-bata/go-prompt v0.2.3 h1:jjCS+QhG/sULBhAaBdjb2PlMRVaKXQgn+4yzaauvs2s=
@@ -116,7 +116,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
116116
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
117117
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
118118
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
119-
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
120119
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
121120
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
122121
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
@@ -141,15 +140,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
141140
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
142141
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
143142
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
144-
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
145143
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
146144
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
147145
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
148146
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
149147
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
150148
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
151149
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
152-
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
153150
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
154151
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
155152
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
@@ -160,7 +157,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
160157
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
161158
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
162159
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
163-
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
164160
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
165161
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
166162
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

0 commit comments

Comments
 (0)