Skip to content

Commit

Permalink
POA-2294 Fix redaction for null values (#47)
Browse files Browse the repository at this point in the history 
As part of this, redaction has been reworked to better use visitors.

Tested locally as well. See the `null` field below.
```
$ ./witness_dump --env local org_295oDMFK8b1yS5dwlSTdgP svc_6NiejyYEVpWfziUXJgovV6 lrn_1H3eLS92zSmgyuiwqkGvB5
method: <
  id: <
    api_type: HTTP_REST
  >
  responses: <
    key: "q2ovkcc4jNQ="
    value: <
      struct: <
        fields: <
          key: "api-key"
          value: <
            primitive: <
              string_value: <
                value: "*REDACTED*"
              >
            >
          >
        >
        fields: <
          key: "empty"
          value: <
            list: <
            >
          >
        >
        fields: <
          key: "null"
          value: <
            optional: <
              none: <
              >
            >
          >
        >
        fields: <
          key: "secret"
          value: <
            primitive: <
              string_value: <
                value: "*REDACTED*"
              >
            >
          >
        >
      >
      meta: <
        http: <
          body: <
            content_type: JSON
            other_type: "application/json"
          >
          response_code: 404
        >
      >
    >
  >
  meta: <
    http: <
      method: "GET"
      path_template: "/"
      processing_latency: 4.242
      obfuscation: NONE
    >
  >
>
```
  • Loading branch information
@liujed
liujed authored Nov 13, 2024
1 parent 0c257a3 commit 024d69a
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 152 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/OneOfOne/xxhash v1.2.8
github.com/Pallinder/go-randomdata v1.2.0
github.com/akitasoftware/akita-ir v0.0.0-20241008173748-ca8e2e3d5db4
github.com/akitasoftware/akita-libs v0.0.0-20241008174225-3b01e02d3430
github.com/akitasoftware/akita-libs v0.0.0-20241113221948-cf2298b3c0fd
github.com/akitasoftware/go-utils v0.0.0-20240213133309-b95d4ace8803
github.com/andybalholm/brotli v1.0.1
github.com/aws/aws-sdk-go-v2 v1.17.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/akitasoftware/akita-ir v0.0.0-20241008173748-ca8e2e3d5db4 h1:jbsA7E68G4dkK+ldQKQNPFIaRojFCn+cEB3gEEUSEU4=
github.com/akitasoftware/akita-ir v0.0.0-20241008173748-ca8e2e3d5db4/go.mod h1:WEWPzhZtxlJnov3MxcqSDiZaHHf00vs3aJwCdt3OwzA=
github.com/akitasoftware/akita-libs v0.0.0-20241008174225-3b01e02d3430 h1:rmeFo57G8voBffbLNBGsqpBtmc7+awRHcWSSk7XdBd4=
github.com/akitasoftware/akita-libs v0.0.0-20241008174225-3b01e02d3430/go.mod h1:5rzkpaJI1sA9CtGELYZpf15oczAVopcyda2WsYG5xno=
github.com/akitasoftware/akita-libs v0.0.0-20241113221948-cf2298b3c0fd h1:ZzV9VguF9u915Alo9p5pWBe/m5R4Pfjq1l/y0Xncrm4=
github.com/akitasoftware/akita-libs v0.0.0-20241113221948-cf2298b3c0fd/go.mod h1:5rzkpaJI1sA9CtGELYZpf15oczAVopcyda2WsYG5xno=
github.com/akitasoftware/go-utils v0.0.0-20240213133309-b95d4ace8803 h1:ebIh/EFuaP8GczzMe8EwVID/blSv5Tej6S8NE4xyarQ=
github.com/akitasoftware/go-utils v0.0.0-20240213133309-b95d4ace8803/go.mod h1:+IOXf7l/QCAQECJzjJwhTp1sBkRoJ6WciZwJezUwBa4=
github.com/akitasoftware/gopacket v1.1.18-0.20240820200020-7289ae956f70 h1:VnU7QLDBwRujpQoHwShs5yu0Ahv1fSalNJa4UijwlmY=
Expand Down
4 changes: 2 additions & 2 deletions trace/backend_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,9 +295,9 @@ func (c *BackendCollector) queueUpload(w *witnessWithInfo) {
if !c.sendWitnessPayloads || !hasOnlyErrorResponses(w.witness.GetMethod()) {
// Obfuscate the original value so type inference engine can use it on the
// backend without revealing the actual value.
c.obfuscator.ObfuscateDataWithZeroValue(w.witness.GetMethod())
c.obfuscator.ZeroAllPrimitivesInMethod(w.witness.GetMethod())
} else {
c.obfuscator.RedactData(w.witness.GetMethod())
c.obfuscator.RedactSensitiveData(w.witness.GetMethod())
}

c.uploadReportBatch.Add(rawReport{
Expand Down
99 changes: 81 additions & 18 deletions trace/backend_collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,14 +519,12 @@ func TestObfuscationConfigs(t *testing.T) {
// Prepare a test cases
streamID := uuid.New()
type testCase struct {
name string
request akinet.HTTPRequest
response akinet.HTTPResponse
expectedWitnesses *api_spec.Witness
}
testCases := []testCase{
{
name: "no sensitive data",
testCases := map[string]testCase{
"no sensitive data": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -589,8 +587,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "sensitive data in header, query param, cookie and URL path",
"sensitive data in header, query param, cookie and URL path": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -687,8 +684,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "sensitive data in body",
"sensitive data in body": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -763,8 +759,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "struct with sensitive keys and values",
"struct with sensitive keys and values": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -869,8 +864,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "list with sensitive keys and values",
"list with sensitive keys and values": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -986,8 +980,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "multipart data, no sensitive data",
"multipart data, no sensitive data": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -1062,8 +1055,7 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
{
name: "multipart data, with sensitive data",
"multipart data, with sensitive data": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Expand Down Expand Up @@ -1152,6 +1144,75 @@ func TestObfuscationConfigs(t *testing.T) {
},
},
},
"null value": {
request: akinet.HTTPRequest{
StreamID: streamID,
Seq: 1204,
Method: "POST",
URL: &url.URL{
Path: "/",
},
Host: "example.com",
Header: map[string][]string{
"Content-Type": {"application/json"},
},
Body: memview.New([]byte(`
{
"null": null
}
`)),
},
response: akinet.HTTPResponse{
StreamID: streamID,
Seq: 1204,
StatusCode: 404,
Header: map[string][]string{
"Content-Type": {"application/json"},
},
Body: memview.New([]byte(`
{
"null": null
}
`)),
},
expectedWitnesses: &pb.Witness{
Method: &pb.Method{
Id: &pb.MethodID{
ApiType: pb.ApiType_HTTP_REST,
},
Args: map[string]*pb.Data{
"sLSDNjJ5umQ=": newTestBodySpecFromStruct(
0,
pb.HTTPBody_JSON,
"application/json",
map[string]*pb.Data{
"null": spec_util.NoneData,
},
),
},
Responses: map[string]*pb.Data{
"2drZdoQw74E=": newTestBodySpecFromStruct(
404,
pb.HTTPBody_JSON,
"application/json",
map[string]*pb.Data{
"null": spec_util.NoneData,
},
),
},
Meta: &pb.MethodMeta{
Meta: &pb.MethodMeta_Http{
Http: &pb.HTTPMethodMeta{
Method: "POST",
PathTemplate: "/",
Host: "example.com",
Obfuscation: api_spec.HTTPMethodMeta_NONE,
},
},
},
},
},
},
}

// Setup for running tests
Expand All @@ -1167,8 +1228,10 @@ func TestObfuscationConfigs(t *testing.T) {
AnyTimes().
Return(nil)

for i, testCase := range testCases {
fmt.Println("Running test case: ", testCase.name)
i := -1
for name, testCase := range testCases {
i++
fmt.Println("Running test case: ", name)

req := akinet.ParsedNetworkTraffic{Content: testCase.request}
resp := akinet.ParsedNetworkTraffic{Content: testCase.response}
Expand Down
Loading

0 comments on commit 024d69a

Please sign in to comment.