Skip to content

Commit caf52d5

Browse files
authored
Remove managed fields (#203)
* Drop managed fields from objects This adds weight to the agent uploads that we don't need Signed-off-by: Charlie Egan <[email protected]> * Remove surplus update of metadata Signed-off-by: Charlie Egan <[email protected]>
1 parent 02338a2 commit caf52d5

File tree

4 files changed

+67
-40
lines changed

4 files changed

+67
-40
lines changed

go.mod

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,8 @@ require (
3333
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
3434
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
3535
golang.org/x/sys v0.0.0-20201005172224-997123666555 // indirect
36-
golang.org/x/text v0.3.3
3736
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
3837
google.golang.org/api v0.30.0
39-
google.golang.org/appengine v1.6.6
4038
gopkg.in/d4l3k/messagediff.v1 v1.2.1
4139
gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
4240
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect

go.sum

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
101101
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
102102
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
103103
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
104-
github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=
105-
github.com/aws/aws-sdk-go v1.25.30/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
106104
github.com/aws/aws-sdk-go v1.34.10 h1:VU78gcf/3wA4HNEDCHidK738l7K0Bals4SJnfnvXOtY=
107105
github.com/aws/aws-sdk-go v1.34.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
108106
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -295,8 +293,6 @@ github.com/jarcoal/httpmock v1.0.1 h1:OXIOrglWeSllwHQGJ5X4PX4hFZK1DPCXSJVhMSJacg
295293
github.com/jarcoal/httpmock v1.0.1/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
296294
github.com/jetstack/version-checker v0.2.1 h1:fTkj1ztb6WP4gMQTROHYAUHJMFMuZOBRRLMrZa0bed4=
297295
github.com/jetstack/version-checker v0.2.1/go.mod h1:908YLy8SWIJMJoYQpZf2QIRAvBvZ2WM0ylNiCrK8Vvg=
298-
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
299-
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
300296
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
301297
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
302298
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -327,6 +323,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
327323
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
328324
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
329325
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
326+
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
330327
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
331328
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
332329
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
@@ -363,6 +360,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
363360
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
364361
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
365362
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
363+
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
366364
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
367365
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
368366
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -760,6 +758,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
760758
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
761759
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
762760
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
761+
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
763762
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
764763
gopkg.in/d4l3k/messagediff.v1 v1.2.1 h1:70AthpjunwzUiarMHyED52mj9UwtAnE89l1Gmrt3EU0=
765764
gopkg.in/d4l3k/messagediff.v1 v1.2.1/go.mod h1:EUzikiKadqXWcD1AzJLagx0j/BeeWGtn++04Xniyg44=

pkg/datagatherer/k8s/dynamic.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,16 +173,18 @@ func redactList(list *unstructured.UnstructuredList) error {
173173
if err != nil {
174174
return errors.WithStack(err)
175175
}
176+
177+
object := list.Items[i]
178+
176179
for _, gvk := range gvks {
177180
// If this item is a Secret then we need to redact it.
178181
if gvk.Kind == "Secret" && (gvk.Group == "core" || gvk.Group == "") {
179-
secret := list.Items[i]
180182

181183
// If the secret is a tls secret, we redact all data other then
182184
// the tls.crt and ca.crt. This is because we need to inspect
183185
// the certificate to make recommendations.
184-
if secret.Object["type"] == "kubernetes.io/tls" {
185-
secretData, ok := secret.Object["data"].(map[string]interface{})
186+
if object.Object["type"] == "kubernetes.io/tls" {
187+
secretData, ok := object.Object["data"].(map[string]interface{})
186188
if ok {
187189
for k := range secretData {
188190
// Only these two keys will be sent, all others are
@@ -193,14 +195,14 @@ func redactList(list *unstructured.UnstructuredList) error {
193195
}
194196
} else {
195197
// If secret is not string mapping, redact all secret data
196-
secret.Object["data"] = map[string]interface{}{}
198+
object.Object["data"] = map[string]interface{}{}
197199
}
198200
} else {
199201
// Redact all secret data for non-tls secrets
200-
secret.Object["data"] = map[string]interface{}{}
202+
object.Object["data"] = map[string]interface{}{}
201203
}
202204

203-
metadata, metadataPresent := secret.Object["metadata"].(map[string]interface{})
205+
metadata, metadataPresent := object.Object["metadata"].(map[string]interface{})
204206
if metadataPresent {
205207
// Redact last-applied-configuration annotation if set
206208
annotations, present := metadata["annotations"].(map[string]interface{})
@@ -211,12 +213,19 @@ func redactList(list *unstructured.UnstructuredList) error {
211213
}
212214
metadata["annotations"] = annotations
213215
}
214-
secret.Object["metadata"] = metadata
215216
}
216217
// break when the object has been processed as a secret, no
217218
// other kinds have redact modifications
218219
break
219220
}
221+
222+
metadata, metadataPresent := object.Object["metadata"].(map[string]interface{})
223+
if metadataPresent {
224+
// Drop managed fields if set
225+
if _, present := metadata["managedFields"]; present {
226+
delete(metadata, "managedFields")
227+
}
228+
}
220229
}
221230
}
222231
return nil

pkg/datagatherer/k8s/dynamic_test.go

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,31 @@ import (
1616
"k8s.io/utils/diff"
1717
)
1818

19-
func getObject(version, kind, name, namespace string) *unstructured.Unstructured {
19+
func getObject(version, kind, name, namespace string, withManagedFields bool) *unstructured.Unstructured {
20+
metadata := map[string]interface{}{
21+
"name": name,
22+
"namespace": namespace,
23+
}
24+
25+
if withManagedFields {
26+
// []metav1.FieldsV1{} can't be deep copied by fake client so using
27+
// string as example value
28+
metadata["managedFields"] = "set"
29+
}
30+
31+
object := map[string]interface{}{
32+
"apiVersion": version,
33+
"kind": kind,
34+
"metadata": metadata,
35+
}
36+
2037
return &unstructured.Unstructured{
21-
Object: map[string]interface{}{
22-
"apiVersion": version,
23-
"kind": kind,
24-
"metadata": map[string]interface{}{
25-
"name": name,
26-
"namespace": namespace,
27-
},
28-
},
38+
Object: object,
2939
}
3040
}
3141

3242
func getSecret(name, namespace string, data map[string]interface{}, isTLS bool, withLastApplied bool) *unstructured.Unstructured {
33-
object := getObject("v1", "Secret", name, namespace)
43+
object := getObject("v1", "Secret", name, namespace, false)
3444
object.Object["data"] = data
3545

3646
object.Object["type"] = "Opaque"
@@ -113,34 +123,34 @@ func TestDynamicGatherer_Fetch(t *testing.T) {
113123
"only a Foo should be returned if GVR selects foos": {
114124
gvr: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
115125
objects: []runtime.Object{
116-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
117-
getObject("v1", "Service", "testservice", "testns"),
118-
getObject("foobar/v1", "NotFoo", "notfoo", "testns"),
126+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
127+
getObject("v1", "Service", "testservice", "testns", false),
128+
getObject("foobar/v1", "NotFoo", "notfoo", "testns", false),
119129
},
120130
expected: asUnstructuredList(
121-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
131+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
122132
),
123133
},
124134
"only Foos in the specified namespace should be returned": {
125135
gvr: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
126136
namespaces: []string{"testns"},
127137
objects: []runtime.Object{
128-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
129-
getObject("foobar/v1", "Foo", "testfoo", "nottestns"),
138+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
139+
getObject("foobar/v1", "Foo", "testfoo", "nottestns", false),
130140
},
131141
expected: asUnstructuredList(
132-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
142+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
133143
),
134144
},
135145
"Foos in different namespaces should be returned if no namespace field is set": {
136146
gvr: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
137147
objects: []runtime.Object{
138-
getObject("foobar/v1", "Foo", "testfoo", "testns1"),
139-
getObject("foobar/v1", "Foo", "testfoo", "testns2"),
148+
getObject("foobar/v1", "Foo", "testfoo", "testns1", false),
149+
getObject("foobar/v1", "Foo", "testfoo", "testns2", false),
140150
},
141151
expected: asUnstructuredList(
142-
getObject("foobar/v1", "Foo", "testfoo", "testns1"),
143-
getObject("foobar/v1", "Foo", "testfoo", "testns2"),
152+
getObject("foobar/v1", "Foo", "testfoo", "testns1", false),
153+
getObject("foobar/v1", "Foo", "testfoo", "testns2", false),
144154
),
145155
},
146156
"Secret resources should have data removed": {
@@ -185,13 +195,24 @@ func TestDynamicGatherer_Fetch(t *testing.T) {
185195
gvr: schema.GroupVersionResource{Group: "foobar", Version: "v1", Resource: "foos"},
186196
namespaces: []string{"testns", "testns2"},
187197
objects: []runtime.Object{
188-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
189-
getObject("foobar/v1", "Foo", "testfoo2", "testns2"),
190-
getObject("foobar/v1", "Foo", "testfoo3", "nottestns"),
198+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
199+
getObject("foobar/v1", "Foo", "testfoo2", "testns2", false),
200+
getObject("foobar/v1", "Foo", "testfoo3", "nottestns", false),
201+
},
202+
expected: asUnstructuredList(
203+
getObject("foobar/v1", "Foo", "testfoo", "testns", false),
204+
getObject("foobar/v1", "Foo", "testfoo2", "testns2", false),
205+
),
206+
},
207+
"Resources should have managed fields removed": {
208+
gvr: schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
209+
objects: []runtime.Object{
210+
getObject("apps/v1", "Deployment", "foo1", "testns", false),
211+
getObject("apps/v1", "Deployment", "foo2", "testns", true),
191212
},
192213
expected: asUnstructuredList(
193-
getObject("foobar/v1", "Foo", "testfoo", "testns"),
194-
getObject("foobar/v1", "Foo", "testfoo2", "testns2"),
214+
getObject("apps/v1", "Deployment", "foo1", "testns", false),
215+
getObject("apps/v1", "Deployment", "foo2", "testns", false),
195216
),
196217
},
197218
// Note that we can't test use of fieldSelector to exclude namespaces
@@ -217,7 +238,7 @@ func TestDynamicGatherer_Fetch(t *testing.T) {
217238
if err == nil && test.err {
218239
t.Errorf("expected to get an error but didn't get one")
219240
}
220-
if diff, equal := messagediff.PrettyDiff(res, test.expected); !equal {
241+
if diff, equal := messagediff.PrettyDiff(test.expected, res); !equal {
221242
t.Errorf("\n%s", diff)
222243
}
223244
})

0 commit comments

Comments
 (0)