From f979c17aee4615a5dbe406df56bad394ecfd7572 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sun, 2 May 2021 22:10:01 +0000 Subject: [PATCH 01/27] idea: get mind together about unpacker spec :lightbulb: --- .github/workflows/go.yml | 2 +- unpacker/unpacker.go | 62 ++++++++++++++ unpacker/unpacker_test.go | 168 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 unpacker/unpacker.go create mode 100644 unpacker/unpacker_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 8b4e0d5..db41987 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -24,4 +24,4 @@ jobs: submodules: true - name: Test - run: go test -v -short . + run: go test -v -short ./... diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go new file mode 100644 index 0000000..c6928cc --- /dev/null +++ b/unpacker/unpacker.go @@ -0,0 +1,62 @@ +package unpacker + +import ( + "encoding/json" + "errors" + "fmt" +) + +type Unpacker struct { + Transforms []Transform + Subs Substitution +} + +type Transform struct { + Assign []string `json:"assignKeys,omitempty"` + Items string `json:"arrayItems,omitempty"` + Key string `json:"rewriteKey,omitempty"` + Name string `json:"-"` // the key of the json object + Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) + Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object +} + +type Substitution map[string]interface{} + +func ParseTransforms(transform []byte) ([]Transform, error) { + v := map[string]Transform{} + if err := json.Unmarshal(transform, &v); err != nil { + return nil, err + } + t := make([]Transform, 0, len(v)) + for key, value := range v { + value.Name = key + t = append(t, value) + } + return t, nil +} + +func (u *Unpacker) AddSubs(subs Substitution) { + if u.Subs == nil { + u.Subs = make(Substitution) + } + for key, value := range subs { + u.Subs[key] = value + } +} + +func (u *Unpacker) AddTransform(trans ...Transform) { + u.Transforms = append(u.Transforms, trans...) +} + +func (u Unpacker) Unpack(source []byte) ([]byte, error) { + obj := map[string]interface{}{} + if err := json.Unmarshal(source, &obj); err != nil { + return nil, err + } + index, hasIndex := obj["?"] + delete(obj, "?") + fmt.Printf("Has index: %t: %v\n", hasIndex, index) + + // note the special ? key in the source object (which may or may not be present) + return nil, errors.New("TODO") +} diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go new file mode 100644 index 0000000..eb1d99a --- /dev/null +++ b/unpacker/unpacker_test.go @@ -0,0 +1,168 @@ +package unpacker + +import ( + "bytes" + "encoding/json" + "testing" +) + +var unpackerTests = []UnpackerTest{ + { + Name: "all", + Skip: true, + Input: `{ + "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], + "o": [ + "Widget Company Ltd", + "Making the best widgets", + "%0", + [ + ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], + ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], + ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] + ] + ] + }`, + Subs: `{ + "ceo": "Chief Executive Officer", + "cto": "Chief Technology Officer", + "cmo": "Chief Marketing Officer" + }`, + Trans: `{ + "o": { + "assignKeys": ["n", "s", "w", "e"], + "replacePair": { + "name": "%n", + "strapline": "%s", + "website": "%w", + "employees": "%e" + } + }, + "e" : { + "arrayItems" : "em" + }, + "em": { + "assignKeys": ["n", "p", "b", "l", "t"], + "replacePair": { + "name": "%n", + "position": "%p", + "bio": "%b", + "linkedin": "https://www.linkedin.com/in/%l", + "twitter": "https://www.twitter.com/%t" + } + } + }`, + Output: `{ + "name": "Widget Company Ltd", + "strapline": "Making the best widgets", + "website": "https://www.widgetcompany.com", + "employees": [{ + "name": "Jane Smith", + "position": "Chief Executive Officer", + "bio": "https://www.widgetcompany.com/team/janesmith", + "linkedin": "https://www.linkedin.com/in/janesmith", + "twitter": "https://www.twitter.com/janesmithwidgets" + }, { + "name": "John Wilson", + "position": "Chief Technology Officer", + "bio": "https://www.widgetcompany.com/team/johnwilson", + "linkedin": "https://www.linkedin.com/in/johnwilson", + "twitter": "https://www.twitter.com/jono" + }, { + "name": "Dashna Anand", + "position": "Chief Marketing Officer", + "bio": "https://www.widgetcompany.com/team/dashnaanand", + "linkedin": "https://www.linkedin.com/in/dashnaanand", + "twitter": "https://www.twitter.com/dashnaanand" + }] + }`, + }, { + Name: "rewriteKey", + Skip: true, + Input: `{"f": 1}`, + Trans: `{"f": {"rewriteKey": "foo"}}`, + Output: `{"foo": 1}`, + }, { + Name: "rewriteValue", + Skip: true, + Input: `{"f": 1}`, + Trans: `{"f": {"rewriteValue" : { "x" : "%self", "y" : 2 }}}`, + Output: `{"foo": {"x": 1, "y": 2}}`, + }, { + Name: "replacePair", + Skip: true, + Input: `{"f": 1}`, + Trans: `{"f": {"replacePair": {"x": "%self", "y": 2, "z": 3}}}`, + Output: `{"x" : 1, "y" : 2, "z" : 3}`, + }, { + Name: "subs", + Skip: true, + Input: `{"foo": "%var"}`, + Subs: `{"var": 1}`, + Output: `{"foo": 1}`, + }, { + Name: "assignKeys", + Skip: true, + Input: `{"phonetic": ["alpha", "bravo", "charlie"]}`, + Trans: `{"phonetic": {"assignKeys": ["a", "b", "c"]}}`, + Output: `{"phonetic": {"a": "alpha", "b": "bravo", "c": "charlie"}}`, + }, { + Name: "arrayItems", + Skip: true, + Input: `{"employees": [["Jane Smith", "CEO"], ["Dashna Anand", "CMO"]]}`, + Trans: `{ + "employees": {"arrayItems": "employee"}, + "employee": { + "assignKeys": ["name", "position"], + "replacePair": {"name": "%name", "position": "%position"} + }}`, + Output: `{ + "employees": [ + {"name": "Jane Smith", "position": "CEO"}, + {"name": "Dashna Anand", "position": "CMO"} + ]}`, + }, +} + +type UnpackerTest struct { + Name string + Input string + Subs string + Trans string + Output string + Skip bool +} + +func (test *UnpackerTest) Run(t *testing.T) { + if testing.Short() && test.Skip { + t.Skip("") + } + u := &Unpacker{} + if test.Trans != `` { + trans, err := ParseTransforms([]byte(test.Trans)) + if err != nil { + t.Fatalf("Unable to parse transforms: %v", err) + } + u.AddTransform(trans...) + } + if test.Subs != `` { + subs := make(Substitution) + if err := json.Unmarshal([]byte(test.Subs), &subs); err != nil { + t.Fatalf("Unable to parse substition: %v", err) + } + u.AddSubs(subs) + } + output, err := u.Unpack([]byte(test.Input)) + if err != nil { + t.Fatalf("Unable to unpack: %v", err) + } + if !bytes.Equal(output, []byte(test.Output)) { + t.Fatalf("Unexpected output:\n\tWant: %q\n\tGot: %q", test.Output, string(output)) + } +} + +func TestUnpacker(t *testing.T) { + for _, test := range unpackerTests { + t.Run(test.Name, test.Run) + } +} From e4b23e37cfcaa512950552e104957b981c33ffaf Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 8 May 2021 17:59:14 +0000 Subject: [PATCH 02/27] feat: basic json recursive parser :sparkles: --- unpacker/unpacker.go | 128 +++++++++++++++++++++++++++++++------- unpacker/unpacker_test.go | 32 +++++++++- 2 files changed, 135 insertions(+), 25 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index c6928cc..b3f5f2f 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -1,13 +1,13 @@ package unpacker import ( + "bytes" "encoding/json" - "errors" "fmt" ) type Unpacker struct { - Transforms []Transform + Transforms map[string]Transform Subs Substitution } @@ -15,24 +15,15 @@ type Transform struct { Assign []string `json:"assignKeys,omitempty"` Items string `json:"arrayItems,omitempty"` Key string `json:"rewriteKey,omitempty"` - Name string `json:"-"` // the key of the json object Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object } type Substitution map[string]interface{} -func ParseTransforms(transform []byte) ([]Transform, error) { +func ParseTransforms(transforms []byte) (map[string]Transform, error) { v := map[string]Transform{} - if err := json.Unmarshal(transform, &v); err != nil { - return nil, err - } - t := make([]Transform, 0, len(v)) - for key, value := range v { - value.Name = key - t = append(t, value) - } - return t, nil + return v, json.Unmarshal(transforms, &v) } func (u *Unpacker) AddSubs(subs Substitution) { @@ -44,19 +35,112 @@ func (u *Unpacker) AddSubs(subs Substitution) { } } -func (u *Unpacker) AddTransform(trans ...Transform) { - u.Transforms = append(u.Transforms, trans...) +func (u *Unpacker) AddTransform(key string, trans Transform) { + // TODO: create map if not exists + // TODO: panic if key already exists + u.Transforms[key] = trans } func (u Unpacker) Unpack(source []byte) ([]byte, error) { - obj := map[string]interface{}{} - if err := json.Unmarshal(source, &obj); err != nil { + state := &unpackState{ctx: u} + + // pass one, extract the question mark lookup index + if err := json.Unmarshal(source, state); err != nil { return nil, err } - index, hasIndex := obj["?"] - delete(obj, "?") - fmt.Printf("Has index: %t: %v\n", hasIndex, index) + // TODO: pre-process the question mark values + + // pass two, process the data + state.dec = json.NewDecoder(bytes.NewReader(source)) + v := state.value() + if state.err != nil { + return nil, state.err + } + return json.Marshal(v) +} + +type unpackState struct { + Idx []interface{} `json:"?"` + ctx Unpacker + dec *json.Decoder + err error +} + +func (state unpackState) value() interface{} { + if state.err != nil { + return nil + } + token, err := state.dec.Token() + if err != nil { + state.err = err + return nil + } + switch v := token.(type) { + case json.Delim: + // fmt.Printf("Got a delimiter: %v\n", token) + switch v.String() { + case "{": + return state.object() + case "[": + return state.array() + default: + panic("Unknown delim string: " + v.String()) + } + case string: + // fmt.Printf("Got a string value: %q\n", token) + return state.string(v) + case float64: + // fmt.Printf("Got an int value: %f\n", token) + return v + default: + panic(fmt.Sprintf("Unknown type: %T", token)) + } +} + +func (state unpackState) object() map[string]interface{} { + if state.err != nil { + return nil + } + + // TODO: skip over `?` in top level object (maybe always for now :badpokerface:) + o := map[string]interface{}{} + for state.dec.More() { + k := state.value().(string) + v := state.value() + + o[k] = v + + // TODO: apply transforms to value + } + state.delim('}') + return o +} - // note the special ? key in the source object (which may or may not be present) - return nil, errors.New("TODO") +func (state unpackState) array() []interface{} { + if state.err != nil { + return nil + } + o := []interface{}{} + for state.dec.More() { + o = append(o, state.value()) + } + state.delim(']') + return o +} + +func (state unpackState) string(in string) string { + if state.err != nil { + return "" + } + return in +} + +func (state unpackState) delim(delim json.Delim) { + cl, err := state.dec.Token() + if err != nil { + panic(err) + } + if cl != delim { + panic(fmt.Sprintf("Expected %v; got %v", delim, cl)) + } } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index eb1d99a..aef0cf6 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -121,6 +121,18 @@ var unpackerTests = []UnpackerTest{ {"name": "Jane Smith", "position": "CEO"}, {"name": "Dashna Anand", "position": "CMO"} ]}`, + }, { + Name: "noop", + Input: `{ + "employees": [ + {"name": "Jane Smith", "position": "CEO"}, + {"name": "Dashna Anand", "position": "CMO"} + ]}`, + Output: `{ + "employees": [ + {"name": "Jane Smith", "position": "CEO"}, + {"name": "Dashna Anand", "position": "CMO"} + ]}`, }, } @@ -143,7 +155,7 @@ func (test *UnpackerTest) Run(t *testing.T) { if err != nil { t.Fatalf("Unable to parse transforms: %v", err) } - u.AddTransform(trans...) + u.Transforms = trans } if test.Subs != `` { subs := make(Substitution) @@ -156,9 +168,23 @@ func (test *UnpackerTest) Run(t *testing.T) { if err != nil { t.Fatalf("Unable to unpack: %v", err) } - if !bytes.Equal(output, []byte(test.Output)) { - t.Fatalf("Unexpected output:\n\tWant: %q\n\tGot: %q", test.Output, string(output)) + exp := test.out() + if !bytes.Equal(output, exp) { + t.Fatalf("Unexpected output:\n\tWant: %q\n\tGot: %q", string(exp), string(output)) + } +} + +func (test UnpackerTest) out() []byte { + var obj interface{} + err := json.Unmarshal([]byte(test.Output), &obj) + if err != nil { + panic(err) + } + bits, err := json.Marshal(obj) + if err != nil { + panic(err) } + return bits } func TestUnpacker(t *testing.T) { From 8883216e88aedfeca7e7d915b6a5015f5d969828 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 8 May 2021 18:15:27 +0000 Subject: [PATCH 03/27] support(replaceKey): ensure the most basic transform works :yay: --- unpacker/unpacker.go | 24 ++++++++++++++++++++---- unpacker/unpacker_test.go | 3 +-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index b3f5f2f..c2a73e7 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -107,10 +107,10 @@ func (state unpackState) object() map[string]interface{} { for state.dec.More() { k := state.value().(string) v := state.value() - - o[k] = v - - // TODO: apply transforms to value + if k == "?" { + continue + } + state.transform(o, k, v) } state.delim('}') return o @@ -144,3 +144,19 @@ func (state unpackState) delim(delim json.Delim) { panic(fmt.Sprintf("Expected %v; got %v", delim, cl)) } } + +func (state unpackState) transform(dest map[string]interface{}, key string, value interface{}) { + trans, ok := state.ctx.Transforms[key] + if !ok { + dest[key] = value + return + } + if trans.Key != "" { + // TODO: do last for multi-transforms? + dest[trans.Key] = value + return + } + + // failsafe + dest[key] = value +} diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index aef0cf6..6c31b3c 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -78,7 +78,6 @@ var unpackerTests = []UnpackerTest{ }`, }, { Name: "rewriteKey", - Skip: true, Input: `{"f": 1}`, Trans: `{"f": {"rewriteKey": "foo"}}`, Output: `{"foo": 1}`, @@ -170,7 +169,7 @@ func (test *UnpackerTest) Run(t *testing.T) { } exp := test.out() if !bytes.Equal(output, exp) { - t.Fatalf("Unexpected output:\n\tWant: %q\n\tGot: %q", string(exp), string(output)) + t.Fatalf("Unexpected output:\n\tWant: %s\n\tGot: %s", string(exp), string(output)) } } From 40890a079032bcb88869af7e54e264c4c78d9b35 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sun, 9 May 2021 01:40:19 +0000 Subject: [PATCH 04/27] add(transforms): support most basic of structure transforms :sparkles: --- unpacker/unpacker.go | 118 ++++++++++++++++++++++++++++++-------- unpacker/unpacker_test.go | 11 +--- 2 files changed, 96 insertions(+), 33 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index c2a73e7..604f107 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -53,9 +53,7 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { // pass two, process the data state.dec = json.NewDecoder(bytes.NewReader(source)) v := state.value() - if state.err != nil { - return nil, state.err - } + // TODO: trap errors? return json.Marshal(v) } @@ -63,17 +61,12 @@ type unpackState struct { Idx []interface{} `json:"?"` ctx Unpacker dec *json.Decoder - err error } func (state unpackState) value() interface{} { - if state.err != nil { - return nil - } token, err := state.dec.Token() if err != nil { - state.err = err - return nil + panic(err) } switch v := token.(type) { case json.Delim: @@ -88,7 +81,7 @@ func (state unpackState) value() interface{} { } case string: // fmt.Printf("Got a string value: %q\n", token) - return state.string(v) + return state.string(v, state.ctx.Subs) case float64: // fmt.Printf("Got an int value: %f\n", token) return v @@ -98,11 +91,6 @@ func (state unpackState) value() interface{} { } func (state unpackState) object() map[string]interface{} { - if state.err != nil { - return nil - } - - // TODO: skip over `?` in top level object (maybe always for now :badpokerface:) o := map[string]interface{}{} for state.dec.More() { k := state.value().(string) @@ -117,9 +105,6 @@ func (state unpackState) object() map[string]interface{} { } func (state unpackState) array() []interface{} { - if state.err != nil { - return nil - } o := []interface{}{} for state.dec.More() { o = append(o, state.value()) @@ -128,10 +113,22 @@ func (state unpackState) array() []interface{} { return o } -func (state unpackState) string(in string) string { - if state.err != nil { - return "" +func (state unpackState) string(in string, subz map[string]interface{}) interface{} { + + // fast lookups + for i, v := range state.Idx { + if in == fmt.Sprintf("%%%d", i) { + return v + } } + for key, v := range subz { + if in == "%"+key { + return v + } + } + + // TODO: dot lookups + // TODO: closing % replacements return in } @@ -151,12 +148,83 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu dest[key] = value return } - if trans.Key != "" { - // TODO: do last for multi-transforms? - dest[trans.Key] = value + + // 0. Items(arrayItems) convert array of arrays to an array of objects + if trans.Items != "" { + list, ok := value.([]interface{}) + if !ok { + panic(fmt.Sprintf("arrayItems set need an array; %T", value)) + } + for i, v := range list { + n := map[string]interface{}{} // TODO: preallocate size + state.transform(n, trans.Items, v) + list[i] = n + } + dest[key] = list + return // not much else we can do here + } + + // 1. Assign(assignKeys) converts arrays to objects + if len(trans.Assign) != 0 { + list, ok := value.([]interface{}) + if !ok { + panic("assignKeys for " + key + " but didn't get a list") + } + if len(list) != len(trans.Assign) { + panic(fmt.Sprintf("assignKeys for %s expected %d but got %d keys", key, len(trans.Assign), len(list))) + } + newValue := make(map[string]interface{}, len(list)) + for i, v := range list { + state.transform(newValue, trans.Assign[i], v) // recurse to assign key/values + } + value = newValue + } + + // 2: replacePair and exit + if trans.Return != nil { + ctx := map[string]interface{}{ + "self": value, + } + if m, ok := value.(map[string]interface{}); ok { + for k, v := range m { + ctx[k] = v + } + } + fmt.Printf("Got replacer: %v\n", value) + for k, v := range trans.Return { + // TODO: smarter string resolves + if s, ok := v.(string); ok { + v = state.string(s, ctx) + } + dest[k] = v + } return } - // failsafe + // 3: rewriteValue + if trans.Rewrite != nil { + if m, ok := trans.Rewrite.(map[string]interface{}); ok { + n := map[string]interface{}{} // ensure we don't duplicate the object + ctx := map[string]interface{}{ + "self": value, + } + for k, v := range m { + if s, ok := v.(string); ok { + v = state.string(s, ctx) + } + n[k] = v + } + value = n + } else { + panic(fmt.Sprintf("got an unexpected rewriteValue type: %T", trans.Rewrite)) + } + } + + // 4: rewriteKey, replace key if need be + if trans.Key != "" { + key = trans.Key + } + + // 5: actually assign object dest[key] = value } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 6c31b3c..471da3a 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -83,31 +83,26 @@ var unpackerTests = []UnpackerTest{ Output: `{"foo": 1}`, }, { Name: "rewriteValue", - Skip: true, - Input: `{"f": 1}`, - Trans: `{"f": {"rewriteValue" : { "x" : "%self", "y" : 2 }}}`, + Input: `{"foo": 1}`, + Trans: `{"foo": {"rewriteValue" : { "x" : "%self", "y" : 2 }}}`, Output: `{"foo": {"x": 1, "y": 2}}`, }, { Name: "replacePair", - Skip: true, Input: `{"f": 1}`, Trans: `{"f": {"replacePair": {"x": "%self", "y": 2, "z": 3}}}`, Output: `{"x" : 1, "y" : 2, "z" : 3}`, }, { Name: "subs", - Skip: true, Input: `{"foo": "%var"}`, Subs: `{"var": 1}`, Output: `{"foo": 1}`, }, { Name: "assignKeys", - Skip: true, Input: `{"phonetic": ["alpha", "bravo", "charlie"]}`, Trans: `{"phonetic": {"assignKeys": ["a", "b", "c"]}}`, Output: `{"phonetic": {"a": "alpha", "b": "bravo", "c": "charlie"}}`, }, { Name: "arrayItems", - Skip: true, Input: `{"employees": [["Jane Smith", "CEO"], ["Dashna Anand", "CMO"]]}`, Trans: `{ "employees": {"arrayItems": "employee"}, @@ -146,7 +141,7 @@ type UnpackerTest struct { func (test *UnpackerTest) Run(t *testing.T) { if testing.Short() && test.Skip { - t.Skip("") + t.Skip("manually disabled test") } u := &Unpacker{} if test.Trans != `` { From 7d4061164b523b459dcaaad57fed299fc97a28b3 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sun, 9 May 2021 01:50:21 +0000 Subject: [PATCH 05/27] fix(strings): not a fast impl, but a working impl :yeet: --- unpacker/unpacker.go | 35 ++++++++++++++++++++++++++++++----- unpacker/unpacker_test.go | 1 - 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 604f107..e84e0bc 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "strconv" + "strings" ) type Unpacker struct { @@ -114,17 +116,40 @@ func (state unpackState) array() []interface{} { } func (state unpackState) string(in string, subz map[string]interface{}) interface{} { - - // fast lookups for i, v := range state.Idx { - if in == fmt.Sprintf("%%%d", i) { + key := "%" + strconv.Itoa(i) + if in == key { return v } + if s, ok := v.(string); ok { + if strings.HasSuffix(in, key) { + in = strings.TrimSuffix(in, key) + s + } + if strings.Contains(in, key+" ") { + in = strings.Replace(in, key+" ", s+" ", -1) + } + if strings.Contains(in, key+"%") { + in = strings.Replace(in, key+"%", s, -1) + } + } } - for key, v := range subz { - if in == "%"+key { + for k, v := range subz { + key := "%" + k + if in == key { return v } + if s, ok := v.(string); ok { + if strings.HasSuffix(in, key) { + in = strings.TrimSuffix(in, key) + s + } + if strings.Contains(in, key+" ") { + in = strings.Replace(in, key+" ", s+" ", -1) + } + if strings.Contains(in, key+"%") { + in = strings.Replace(in, key+"%", s, -1) + } + } + } // TODO: dot lookups diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 471da3a..f84dd86 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -9,7 +9,6 @@ import ( var unpackerTests = []UnpackerTest{ { Name: "all", - Skip: true, Input: `{ "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], "o": [ From eac0c3fb8893323c45c2e41299b75968e05c5b48 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 03:25:03 +0000 Subject: [PATCH 06/27] add all the recently announced docs --- unpacker/unpacker_test.go | 1426 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1426 insertions(+) diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index f84dd86..e55c149 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -7,6 +7,7 @@ import ( ) var unpackerTests = []UnpackerTest{ + // Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d { Name: "all", Input: `{ @@ -127,6 +128,1431 @@ var unpackerTests = []UnpackerTest{ {"name": "Dashna Anand", "position": "CMO"} ]}`, }, + + // Source: https://www.unpacker.uk/playground?e=3 + { + Name: "playground-3", + Skip: true, + Input: `{ + "@n": 1, + "o": { + "n": "NUM Example Co", + "s": "Example Strapline", + "c": [ + { + "t": { + "d": "Customer Service", + "v": "+441270123456" + } + }, + { + "fb": { + "v": "examplefacebook" + } + }, + { + "in": { + "v": "exampleinstagram" + } + }, + { + "tw": { + "v": "exampletwitter" + } + } + ] + } + }`, + Subs: `{ + "locale": { + "p": { + "name": "Person" + }, + "gr": { + "name": "Group" + }, + "o": { + "name": "Organisation" + }, + "dp": { + "name": "Department" + }, + "e": { + "name": "Employee" + }, + "lc": { + "name": "Location" + }, + "gp": { + "name": "Group" + }, + "t": { + "name": "Telephone", + "default": "Call" + }, + "sm": { + "name": "SMS", + "default": "Text" + }, + "u": { + "name": "Web URL", + "default": "Click" + }, + "uu": { + "name": "Web URL (http - unsecure)", + "default": "Click" + }, + "g": { + "name": "GPS", + "default": "View Location" + }, + "a": { + "name": "Address", + "default": "View Address" + }, + "fx": { + "name": "Fax", + "default": "Send a fax" + }, + "em": { + "name": "Email", + "default": "Send an email" + }, + "aa": { + "name": "Android App", + "default": "Download the app" + }, + "as": { + "name": "iOS App", + "default": "Download the app" + }, + "bt": { + "name": "Baidu Tieba", + "default": "View Baidu profile" + }, + "fb": { + "name": "Facebook", + "default": "View Facebook profile" + }, + "fs": { + "name": "FourSquare", + "default": "View FourSquare page" + }, + "ft": { + "name": "FaceTime", + "default": "Call with Facetime" + }, + "gh": { + "name": "Github", + "default": "View Github profile" + }, + "im": { + "name": "iMessage", + "default": "Send iMessage" + }, + "in": { + "name": "Instagram", + "default": "View Instagram profile" + }, + "kk": { + "name": "Kik", + "default": "Connect with Kik" + }, + "li": { + "name": "LinkedIn", + "default": "View LinkedIn page" + }, + "ln": { + "name": "Line", + "default": "Connect with Line" + }, + "md": { + "name": "Medium", + "default": "View Medium blog" + }, + "pr": { + "name": "Periscope", + "default": "View Periscope profile" + }, + "pi": { + "name": "Pinterest", + "default": "View Pinterest board" + }, + "qq": { + "name": "QQ", + "default": "View QQ Page" + }, + "qz": { + "name": "Qzone", + "default": "View Qzone Page" + }, + "rd": { + "name": "Reddit", + "default": "View subreddit" + }, + "rn": { + "name": "Renren", + "default": "View Renren profile" + }, + "sc": { + "name": "Soundcloud", + "default": "View Soundcloud page" + }, + "sk": { + "name": "Skype", + "default": "Call with Skype" + }, + "sr": { + "name": "Swarm", + "default": "Connect with Swarm" + }, + "sn": { + "name": "Snapchat", + "default": "Connect with Snapchat" + }, + "sw": { + "name": "Sina Weibo", + "default": "View Weibo page" + }, + "tb": { + "name": "Tumblr", + "default": "View Tumblr blog" + }, + "tl": { + "name": "Telegram", + "default": "Connect with Telegram" + }, + "tw": { + "name": "Twitter", + "default": "View Twitter profile" + }, + "to": { + "name": "Twoo", + "default": "View Twoo page" + }, + "vb": { + "name": "Viber", + "default": "Call with Viber" + }, + "vk": { + "name": "Vkontakte", + "default": "View VK page" + }, + "vm": { + "name": "Vimeo", + "default": "View Vimeo profile" + }, + "wa": { + "name": "Whatsapp", + "default": "Message on Whatsapp" + }, + "wc": { + "name": "WeChat", + "default": "Connect with WeChat" + }, + "xi": { + "name": "Xing", + "default": "View Xing page" + }, + "yt": { + "name": "YouTube", + "default": "View YouTube channel" + }, + "yy": { + "name": "YY", + "default": "View YY page" + } + }, + "AC": "Accounts", + "CS": "Customer Service" + }`, + Trans: `{ + "o": { + "rewriteKey": "organisation", + "assignKeys": [ + "n", + "s", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.o.name", + "name": "%n", + "slogan": "%s", + "contacts": "%c" + } + }, + "p": { + "rewriteKey": "person", + "assignKeys": [ + "n", + "b", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.p.name", + "name": "%n", + "bio": "%b", + "contacts": "%c" + } + }, + "e": { + "rewriteKey": "employee", + "assignKeys": [ + "n", + "r", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.e.name", + "name": "%n", + "role": "%r", + "contacts": "%c" + } + }, + "lc": { + "rewriteKey": "location", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.lc.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "gp": { + "rewriteKey": "group", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.gp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "dp": { + "rewriteKey": "department", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.dp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "a": { + "rewriteKey": "address", + "assignKeys": [ + "al", + "pz", + "co", + "d" + ], + "rewriteValue": { + "description": "%d", + "description_default": "%/subs.locale.a.default", + "lines": "%al", + "postcode": "%pz", + "country": "%co", + "method_type": "core", + "object_display_name": "%/subs.locale.a.name", + "prefix": "" + } + }, + "l": { + "rewriteKey": "link", + "assignKeys": [ + "@L", + "d" + ], + "rewriteValue": { + "@L": "%@L", + "description": "%d" + } + }, + "fb": { + "rewriteKey": "facebook", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fb.name", + "description_default": "%/subs.locale.fb.default", + "description": "%d", + "prefix": "https://www.facebook.com/", + "method_type": "third_party", + "controller": "facebook.com", + "value": "%v" + } + }, + "g": { + "rewriteKey": "gps", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.g.name", + "description_default": "%/subs.locale.g.default", + "description": "%d", + "prefix": "", + "method_type": "core", + "value": "%v" + } + }, + "in": { + "rewriteKey": "instagram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.in.name", + "description_default": "%/subs.locale.in.default", + "description": "%d", + "prefix": "https://www.instagram.com/", + "method_type": "third_party", + "controller": "instagram.com", + "value": "%v" + } + }, + "li": { + "rewriteKey": "linkedin", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.li.name", + "description_default": "%/subs.locale.li.default", + "description": "%d", + "prefix": "https://www.linkedin.com/", + "method_type": "third_party", + "controller": "linkedin.com", + "value": "%v" + } + }, + "yt": { + "rewriteKey": "youtube", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.yt.name", + "description_default": "%/subs.locale.yt.default", + "description": "%d", + "prefix": "https://www.youtube.com/", + "method_type": "third_party", + "controller": "youtube.com", + "value": "%v" + } + }, + "pi": { + "rewriteKey": "pinterest", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.pi.name", + "description_default": "%/subs.locale.pi.default", + "description": "%d", + "prefix": "https://www.pinterest.com/", + "method_type": "third_party", + "controller": "pinterest.com", + "value": "%v" + } + }, + "tw": { + "rewriteKey": "twitter", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.tw.name", + "description_default": "%/subs.locale.tw.default", + "description": "%d", + "prefix": "https://www.twitter.com/", + "method_type": "third_party", + "controller": "twitter.com", + "value": "%v", + "value_prefix": "@" + } + }, + "t": { + "rewriteKey": "telephone", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.t.name", + "description_default": "%/subs.locale.t.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "sm": { + "rewriteKey": "sms", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.sm.name", + "description_default": "%/subs.locale.sm.default", + "description": "%d", + "prefix": "sms:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "em": { + "rewriteKey": "email", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.em.name", + "description_default": "%/subs.locale.em.default", + "description": "%d", + "prefix": "mailto:", + "method_type": "core", + "value": "%v" + } + }, + "fx": { + "rewriteKey": "fax", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fx.name", + "description_default": "%/subs.locale.fx.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v" + } + }, + "u": { + "rewriteKey": "url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.u.name", + "description_default": "%/subs.locale.u.default", + "description": "%d", + "prefix": "https://", + "method_type": "core", + "value": "%v" + } + }, + "uu": { + "rewriteKey": "unsecure_url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.uu.name", + "description_default": "%/subs.locale.uu.default", + "description": "%d", + "prefix": "http://", + "method_type": "core", + "value": "%v" + } + }, + "av": { + "rewriteKey": "available" + }, + "tz": { + "rewriteKey": "time_zone_location" + }, + "i": { + "rewriteKey": "introduction" + }, + "ac": { + "rewriteKey": "access" + }, + "aa": { + "rewriteKey": "android-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.aa.name", + "description_default": "locale.aa.default", + "prefix": "https://play.google.com/store/apps/details?id=", + "method_type": "third_party", + "controller": "play.google.com" + } + }, + "as": { + "rewriteKey": "ios-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.as.name", + "description_default": "locale.as.default", + "prefix": "https://itunes.apple.com/app/", + "method_type": "third_party", + "controller": "apps.apple.com" + } + }, + "bt": { + "rewriteKey": "baidu_tieba", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.bt.name", + "description_default": "locale.bt.default", + "prefix": "https://tieba.baidu.com/", + "method_type": "third_party", + "controller": "tieba.baidu.com" + } + }, + "fs": { + "rewriteKey": "foursquare", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.fs.name", + "description_default": "locale.fs.default", + "prefix": "https://www.foursquare.com/", + "method_type": "third_party", + "controller": "foursquare.com" + } + }, + "ft": { + "rewriteKey": "facetime", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ft.name", + "description_default": "locale.ft.default", + "prefix": "facetime://", + "method_type": "third_party", + "controller": "facetime@apple.com" + } + }, + "gh": { + "rewriteKey": "github", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.gh.name", + "description_default": "locale.gh.default", + "prefix": "https://www.github.com/", + "method_type": "third_party", + "controller": "github.com" + } + }, + "im": { + "rewriteKey": "imessage", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.im.name", + "description_default": "locale.im.default", + "prefix": "imessage://", + "method_type": "third_party", + "controller": "imessage@apple.com" + } + }, + "kk": { + "rewriteKey": "kik", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.kk.name", + "description_default": "locale.kk.default", + "prefix": "https://www.kik.com/u/", + "method_type": "third_party", + "controller": "kik.com" + } + }, + "ln": { + "rewriteKey": "line", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ln.name", + "description_default": "locale.ln.default", + "prefix": "line://", + "method_type": "third_party", + "controller": "line.me" + } + }, + "md": { + "rewriteKey": "medium", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.md.name", + "description_default": "locale.md.default", + "prefix": "https://www.medium.com/", + "method_type": "third_party", + "controller": "medium.com" + } + }, + "pr": { + "rewriteKey": "periscope", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.pr.name", + "description_default": "locale.pr.default", + "prefix": "https://www.periscope.tv/", + "method_type": "third_party", + "controller": "periscope.tv" + } + }, + "qq": { + "rewriteKey": "qq", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qq.name", + "description_default": "locale.qq.default", + "prefix": "https://www.qq.com/", + "method_type": "third_party", + "controller": "qq.com" + } + }, + "qz": { + "rewriteKey": "qzone", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qz.name", + "description_default": "locale.qz.default", + "prefix": "https://www.qzone.com/", + "method_type": "third_party", + "controller": "qzone.com" + } + }, + "rd": { + "rewriteKey": "reddit", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rd.name", + "description_default": "locale.rd.default", + "prefix": "https://www.reddit.com/r/", + "method_type": "third_party", + "controller": "reddit.com" + } + }, + "rn": { + "rewriteKey": "renren", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rn.name", + "description_default": "locale.rn.default", + "prefix": "https://www.renren.com/", + "method_type": "third_party", + "controller": "renren.com" + } + }, + "sc": { + "rewriteKey": "soundcloud", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sc.name", + "description_default": "locale.sc.default", + "prefix": "https://www.soundcloud.com/", + "method_type": "third_party", + "controller": "soundcloud.com" + } + }, + "sk": { + "rewriteKey": "skype", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sk.name", + "description_default": "locale.sk.default", + "prefix": "skype:", + "method_type": "third_party", + "controller": "skype.com" + } + }, + "sr": { + "rewriteKey": "swarm", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sr.name", + "description_default": "locale.sr.default", + "prefix": "https://www.swarmapp.com/", + "method_type": "third_party", + "controller": "swarmapp.com" + } + }, + "sn": { + "rewriteKey": "snapchat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sn.name", + "description_default": "locale.sn.default", + "prefix": "snapchat://add/", + "method_type": "third_party", + "controller": "snapchat.com" + } + }, + "sw": { + "rewriteKey": "sina-weibo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sw.name", + "description_default": "locale.sw.default", + "prefix": "https://www.weibo.com/", + "method_type": "third_party", + "controller": "weibo.com" + } + }, + "tb": { + "rewriteKey": "tumblr", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tb.name", + "description_default": "locale.tb.default", + "prefix": "https://.tumblr.com/", + "method_type": "third_party", + "controller": "tumblr.com" + } + }, + "tl": { + "rewriteKey": "telegram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tl.name", + "description_default": "locale.tl.default", + "prefix": "https://www.telegram.me/", + "method_type": "third_party", + "controller": "telegram.com" + } + }, + "to": { + "rewriteKey": "twoo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.to.name", + "description_default": "locale.to.default", + "prefix": "https://www.twoo.com/", + "method_type": "third_party", + "controller": "twoo.com" + } + }, + "vb": { + "rewriteKey": "viber", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vb.name", + "description_default": "locale.vb.default", + "prefix": "https://www.viber.com/", + "method_type": "third_party", + "controller": "viber.com" + } + }, + "vk": { + "rewriteKey": "vkontakte", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vk.name", + "description_default": "locale.vk.default", + "prefix": "https://www.vk.com/", + "method_type": "third_party", + "controller": "vk.com" + } + }, + "vm": { + "rewriteKey": "vimeo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vm.name", + "description_default": "locale.vm.default", + "prefix": "https://www.vimeo.com/", + "method_type": "third_party", + "controller": "vimeo.com" + } + }, + "wa": { + "rewriteKey": "whatsapp", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wa.name", + "description_default": "locale.wa.default", + "prefix": "whatsapp://", + "method_type": "third_party", + "controller": "whatsapp.com" + } + }, + "wc": { + "rewriteKey": "wechat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wc.name", + "description_default": "locale.wc.default", + "prefix": "https://www.wechat.com/", + "method_type": "third_party", + "controller": "wechat.com" + } + }, + "xi": { + "rewriteKey": "xing", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.xi.name", + "description_default": "locale.xi.default", + "prefix": "https://www.xing.com/", + "method_type": "third_party", + "controller": "xing.com" + } + }, + "yy": { + "rewriteKey": "yy", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.yy.name", + "description_default": "locale.yy.default", + "prefix": "https://www.yy.com/", + "method_type": "third_party", + "controller": "yy.com" + } + } + }`, + Output: `{ + "@n": 1, + "organisation": { + "object_display_name": "Organisation", + "name": "NUM Example Co", + "slogan": "Example Strapline", + "contacts": [ + { + "telephone": { + "object_display_name": "Telephone", + "description_default": "Call", + "description": "Customer Service", + "prefix": "tel:", + "method_type": "core", + "value": "+441270123456", + "hours": null + } + }, + { + "facebook": { + "object_display_name": "Facebook", + "description_default": "View Facebook profile", + "description": null, + "prefix": "https://www.facebook.com/", + "method_type": "third_party", + "controller": "facebook.com", + "value": "examplefacebook" + } + }, + { + "instagram": { + "object_display_name": "Instagram", + "description_default": "View Instagram profile", + "description": null, + "prefix": "https://www.instagram.com/", + "method_type": "third_party", + "controller": "instagram.com", + "value": "exampleinstagram" + } + }, + { + "twitter": { + "object_display_name": "Twitter", + "description_default": "View Twitter profile", + "description": null, + "prefix": "https://www.twitter.com/", + "method_type": "third_party", + "controller": "twitter.com", + "value": "exampletwitter", + "value_prefix": "@" + } + } + ] + } + }`, + }, + + // Source: https://www.unpacker.uk/playground?e=2 + { + Name: "playground-2", + Input: `{ + "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], + "o": [ + "Widget Company Ltd", + "Making the best widgets", + "%0", + [ + ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], + ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], + ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] + ] + ] + }`, + Subs: `{ + "ceo": "Chief Executive Officer", + "cto": "Chief Technology Officer", + "cmo": "Chief Marketing Officer" + }`, + Trans: `{ + "o": { + "assignKeys": ["n", "s", "w", "e"], + "replacePair": { + "name": "%n", + "strapline": "%s", + "website": "%w", + "employees": "%e" + } + }, + "em": { + "replacePair": { + "name": "%n", + "position": "%p", + "bio": "%b", + "linkedin": "https://www.linkedin.com/in/%l", + "twitter": "https://www.twitter.com/%t" + } + } + }`, + Output: `{ + "name": "Widget Company Ltd", + "strapline": "Making the best widgets", + "website": "https://www.widgetcompany.com", + "employees": [ + [ + "Jane Smith", + "Chief Executive Officer", + "https://www.widgetcompany.com/team/janesmith", + "janesmith", + "janesmithwidgets" + ], + [ + "John Wilson", + "Chief Technology Officer", + "https://www.widgetcompany.com/team/johnwilson", + "johnwilson", + "jono" + ], + [ + "Dashna Anand", + "Chief Marketing Officer", + "https://www.widgetcompany.com/team/dashnaanand", + "dashnaanand", + "dashnaanand" + ] + ] + }`, + }, + + // Source: https://www.unpacker.uk/playground?e=1 + { + Name: "playground-1", + Input: `{ + "?": ["abccompany", "+4412345678"], + "c": "ABC Company Ltd", + "t": [ + {"l": "%ac", "n": "%1%90"}, + {"l": "%cs", "n": "%1%89"} + ], + "tw": "/%0", + "i": "/%0%pics" + }`, + Subs: `{"cs": "Customer Service", "ac": "Accounts"}`, + Trans: `{ + "c": {"rewriteKey": "coname"}, + "t": {"rewriteKey": "telephone"}, + "l": {"rewriteKey": "label"}, + "n": {"rewriteKey": "number"}, + "tw": {"rewriteKey": "twitter"}, + "i": {"rewriteKey": "instagram"} + }`, + Output: `{ + "coname": "ABC Company Ltd", + "telephone": [ + {"label": "Accounts", "number": "+441234567890"}, + {"label": "Customer Service", "number": "+441234567889"} + ], + "twitter": "/abccompany", + "instagram": "/abccompanypics" + }`, + }, + + // Source: https://www.unpacker.uk/specification#variable-index + { + Name: "variable-index-1", + Skip: true, + Input: `{"?": [1], "a": "%0%"}`, + Output: `{"a": 1}`, + }, + { + Name: "variable-index-2", + Skip: true, + Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0%", "y": "%1%"}`, + Output: `{"x": ["a", "b"], "y": {"a": "alpha", "b": "bravo"}}`, + }, + { + Name: "variable-index-3", + Skip: true, + Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0.0%", "y": "%1.a%"}`, + Output: `{"x": "a", "y": "alpha"}`, + }, + + // Source: https://www.unpacker.uk/specification#substitution-object + { + Name: "sub-1", + Skip: true, + Input: `{"this": "%a%", "that": "%b%"}`, + Subs: `{"a": 1, "b": 2}`, + Output: `{"this": 1, "that": 2}`, + }, + { + Name: "sub-2", + Skip: true, + Input: `{"this": "%a.x%", "that": "%b.0%"}`, + Subs: `{"a": {"x": "xray", "y": "yankee"}, "b": [1, 2]}`, + Output: `{"this": "xray", "that": 1}`, + }, + + // Source: https://www.unpacker.uk/specification#transformation-object + { + Name: "trans-1", + Input: `{ + "t": 1, + "u": { + "t": 2 + }, + "v": { + "w": { + "t": 3 + } + } + }`, + Trans: `{ + "t": { + "rewriteKey": "testing" + } + }`, + Output: `{ + "testing": 1, + "u": { + "testing": 2 + }, + "v": { + "w": { + "testing": 3 + } + } + } + `, + }, + { + Name: "trans-2", + Skip: true, + Input: `{ + "t": 1, + "u": { + "t": 2 + }, + "v": { + "w": { + "t": 3 + } + } + }`, + Trans: `{ + "t": { + "rewriteKey": "testing" + }, + "u": { + "t": { + "rewriteKey": "testing2" + } + }, + "v": { + "w": { + "t": { + "rewriteKey": "testing3" + } + } + } + }`, + Output: `{ + "testing": 1, + "u": { + "testing2": 2 + }, + "v": { + "w": { + "testing3": 3 + } + } + }`, + }, + { + Name: "trans-3", + Input: `{ + "t": 1 + }`, + Trans: `{ + "t": { + "rewriteKey": "testing" + } + }`, + Output: `{"testing": 1}`, + }, + { + Name: "trans-4", + Skip: true, + Input: `{"t": "me"}`, + Trans: `{ + "t": { + "rewriteKey": "twitter", + "rewriteValue": "twitter.com/%self" + } + }`, + Output: `{"twitter": "twitter.com/me"}`, + }, + { + Name: "trans-5", + Input: `{"t": "me"}`, + Trans: `{ + "t": { + "rewriteKey": "twitter", + "rewriteValue": { + "cta": "Follow on Twitter", + "url": "twitter.com/%self" + } + } + }`, + Output: `{ + "twitter": { + "cta": "Follow on Twitter", + "url": "twitter.com/me" + } + }`, + }, + { + Name: "trans-6", + Input: `{"t": "me"}`, + Trans: `{ + "t": { + "replacePair": { + "cta": "Follow on Twitter", + "url": "twitter.com/%self" + } + } + }`, + Output: `{"cta": "Follow on Twitter", "url": "twitter.com/me"}`, + }, + { + Name: "trans-7", + Input: `{"phonetic": ["alpha", "bravo", "charlie"]}`, + Trans: `{"phonetic": {"assignKeys": ["a", "b", "c"]}}`, + Output: `{ + "phonetic": { + "a": "alpha", + "b": "bravo", + "c": "charlie" + } + }`, + }, + { + Name: "trans-8", + Input: `{ + "employees": [ + ["Jane Smith", "CEO"], + ["Dashna Anand", "CMO"] + ] + }`, + Trans: `{ + "employees": { + "arrayItems": "employee" + }, + "employee": { + "assignKeys": [ + "name", + "position" + ], + "replacePair": { + "name": "%name", + "position": "%position" + } + } + }`, + Output: `{"employees": [ + {"name": "Jane Smith", "position": "CEO"}, + {"name": "Dashna Anand", "position": "CMO"} + ]}`, + }, + + // Source: https://www.unpacker.uk/specification#referencing + { + Name: "ref-1", + Input: `{ + "?": ["test"], + "a": "%0%", + "b": "this is a %0", + "c": "this is another %0 of referencing", + "d": "once again %0%ing it" + }`, + Output: `{ + "a": "test", + "b": "this is a test", + "c": "this is another test of referencing", + "d": "once again testing it" + }`, + }, + { + Name: "ref-2", + Skip: true, + Input: `{ + "?": [ + ["a", "b"], + {"a": "alpha", "b": "bravo"} + ], + "x": "%0.0", + "y": "%1.a", + "z": "%letters.0" + }`, + Subs: `{"letters": ["a", "b"]}`, + Output: `{"x": "a", "y": "alpha", "z": "a"}`, + }, + { + Name: "ref-3", + Skip: true, + Input: `{"a": {"b": {"c": 1}}}`, + Trans: `{"a": {"rewriteValue": "%b.c"}}`, + Output: `{"a": 1}`, + }, + { + Name: "ref-4", + Skip: true, + Input: `{ + "a": ["alpha", "bravo", "charlie"] + }`, + Trans: `{ + "a": { + "assignKeys": ["a", "b", "c"], + "rewriteValue": "%c" + } + }`, + Output: `{"a": "charlie"}`, + }, + { + Name: "ref-5", + Skip: true, + Input: `{"a": 1, "b": 2, "c": 3}`, + Trans: `{ + "a": {"rewriteValue": "%/compact.c"}, + "b": {"rewriteValue": "%/subs.x"} + }`, + Subs: `{"x": 4}`, + Output: `{"a": 3, "b": 4, "c": 3}`, + }, } type UnpackerTest struct { From a3c07c414642d85b0117f891dabf5811fd8bdf84 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 03:55:56 +0000 Subject: [PATCH 07/27] fix up a few sample tests --- unpacker/unpacker.go | 17 +++++++++------ unpacker/unpacker_test.go | 45 ++++++++------------------------------- 2 files changed, 19 insertions(+), 43 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index e84e0bc..579f8cb 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -118,7 +118,7 @@ func (state unpackState) array() []interface{} { func (state unpackState) string(in string, subz map[string]interface{}) interface{} { for i, v := range state.Idx { key := "%" + strconv.Itoa(i) - if in == key { + if in == key || in == key+"%" { return v } if s, ok := v.(string); ok { @@ -135,7 +135,7 @@ func (state unpackState) string(in string, subz map[string]interface{}) interfac } for k, v := range subz { key := "%" + k - if in == key { + if in == key || in == key+"%" { return v } if s, ok := v.(string); ok { @@ -193,7 +193,8 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu if len(trans.Assign) != 0 { list, ok := value.([]interface{}) if !ok { - panic("assignKeys for " + key + " but didn't get a list") + fmt.Print("assignKeys for " + key + " but didn't get a list") + return } if len(list) != len(trans.Assign) { panic(fmt.Sprintf("assignKeys for %s expected %d but got %d keys", key, len(trans.Assign), len(list))) @@ -215,7 +216,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu ctx[k] = v } } - fmt.Printf("Got replacer: %v\n", value) + // fmt.Printf("Got replacer: %v\n", value) for k, v := range trans.Return { // TODO: smarter string resolves if s, ok := v.(string); ok { @@ -228,11 +229,11 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // 3: rewriteValue if trans.Rewrite != nil { + ctx := map[string]interface{}{ + "self": value, + } if m, ok := trans.Rewrite.(map[string]interface{}); ok { n := map[string]interface{}{} // ensure we don't duplicate the object - ctx := map[string]interface{}{ - "self": value, - } for k, v := range m { if s, ok := v.(string); ok { v = state.string(s, ctx) @@ -240,6 +241,8 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu n[k] = v } value = n + } else if s, ok := trans.Rewrite.(string); ok { + value = state.string(s, ctx) } else { panic(fmt.Sprintf("got an unexpected rewriteValue type: %T", trans.Rewrite)) } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index e55c149..c23c246 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1284,13 +1284,11 @@ var unpackerTests = []UnpackerTest{ // Source: https://www.unpacker.uk/specification#variable-index { Name: "variable-index-1", - Skip: true, Input: `{"?": [1], "a": "%0%"}`, Output: `{"a": 1}`, }, { Name: "variable-index-2", - Skip: true, Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0%", "y": "%1%"}`, Output: `{"x": ["a", "b"], "y": {"a": "alpha", "b": "bravo"}}`, }, @@ -1304,7 +1302,6 @@ var unpackerTests = []UnpackerTest{ // Source: https://www.unpacker.uk/specification#substitution-object { Name: "sub-1", - Skip: true, Input: `{"this": "%a%", "that": "%b%"}`, Subs: `{"a": 1, "b": 2}`, Output: `{"this": 1, "that": 2}`, @@ -1322,32 +1319,15 @@ var unpackerTests = []UnpackerTest{ Name: "trans-1", Input: `{ "t": 1, - "u": { - "t": 2 - }, - "v": { - "w": { - "t": 3 - } - } - }`, - Trans: `{ - "t": { - "rewriteKey": "testing" - } + "u": {"t": 2}, + "v": {"w": {"t": 3}} }`, + Trans: `{"t": {"rewriteKey": "testing"}}`, Output: `{ "testing": 1, - "u": { - "testing": 2 - }, - "v": { - "w": { - "testing": 3 - } - } - } - `, + "u": {"testing": 2}, + "v": {"w": {"testing": 3}} + }`, }, { Name: "trans-2", @@ -1393,20 +1373,13 @@ var unpackerTests = []UnpackerTest{ }`, }, { - Name: "trans-3", - Input: `{ - "t": 1 - }`, - Trans: `{ - "t": { - "rewriteKey": "testing" - } - }`, + Name: "trans-3", + Input: `{"t": 1}`, + Trans: `{"t": {"rewriteKey": "testing"}}`, Output: `{"testing": 1}`, }, { Name: "trans-4", - Skip: true, Input: `{"t": "me"}`, Trans: `{ "t": { From a955bb3594432755e5bb51fc35e0bec321f61233 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 04:50:35 +0000 Subject: [PATCH 08/27] fixing sub-object parsing --- unpacker/unpacker.go | 42 ++++++++++++++++++++++++++++++++------- unpacker/unpacker_test.go | 5 ----- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 579f8cb..9cd90a4 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -44,13 +44,17 @@ func (u *Unpacker) AddTransform(key string, trans Transform) { } func (u Unpacker) Unpack(source []byte) ([]byte, error) { - state := &unpackState{ctx: u} + state := &unpackState{ctx: u, mem: make(map[string]interface{})} // pass one, extract the question mark lookup index if err := json.Unmarshal(source, state); err != nil { return nil, err } - // TODO: pre-process the question mark values + + // memoize string replacements + augment("", state.Idx, state.mem) + augment("", u.Subs, state.mem) + // fmt.Printf("Values: %#v\n", state.mem) // pass two, process the data state.dec = json.NewDecoder(bytes.NewReader(source)) @@ -63,6 +67,7 @@ type unpackState struct { Idx []interface{} `json:"?"` ctx Unpacker dec *json.Decoder + mem map[string]interface{} } func (state unpackState) value() interface{} { @@ -83,7 +88,7 @@ func (state unpackState) value() interface{} { } case string: // fmt.Printf("Got a string value: %q\n", token) - return state.string(v, state.ctx.Subs) + return state.string(v, nil) case float64: // fmt.Printf("Got an int value: %f\n", token) return v @@ -116,8 +121,8 @@ func (state unpackState) array() []interface{} { } func (state unpackState) string(in string, subz map[string]interface{}) interface{} { - for i, v := range state.Idx { - key := "%" + strconv.Itoa(i) + for k, v := range state.mem { + key := "%" + k if in == key || in == key+"%" { return v } @@ -149,7 +154,6 @@ func (state unpackState) string(in string, subz map[string]interface{}) interfac in = strings.Replace(in, key+"%", s, -1) } } - } // TODO: dot lookups @@ -193,7 +197,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu if len(trans.Assign) != 0 { list, ok := value.([]interface{}) if !ok { - fmt.Print("assignKeys for " + key + " but didn't get a list") + fmt.Println("assignKeys for " + key + " but didn't get a list") return } if len(list) != len(trans.Assign) { @@ -232,6 +236,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu ctx := map[string]interface{}{ "self": value, } + augment("", value, ctx) if m, ok := trans.Rewrite.(map[string]interface{}); ok { n := map[string]interface{}{} // ensure we don't duplicate the object for k, v := range m { @@ -256,3 +261,26 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // 5: actually assign object dest[key] = value } + +func augment(prefix string, thing interface{}, target map[string]interface{}) { + switch value := thing.(type) { + case []interface{}: + for i, v := range value { + x := prefix + strconv.Itoa(i) + target[x] = v + augment(x+".", v, target) + } + case map[string]interface{}: + for k, v := range value { + x := prefix + k + target[x] = v + augment(x+".", v, target) + } + case Substitution: // map[string]interface{} + for k, v := range value { + x := prefix + k + target[x] = v + augment(x+".", v, target) + } + } +} diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index c23c246..fc74b37 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1294,7 +1294,6 @@ var unpackerTests = []UnpackerTest{ }, { Name: "variable-index-3", - Skip: true, Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0.0%", "y": "%1.a%"}`, Output: `{"x": "a", "y": "alpha"}`, }, @@ -1308,7 +1307,6 @@ var unpackerTests = []UnpackerTest{ }, { Name: "sub-2", - Skip: true, Input: `{"this": "%a.x%", "that": "%b.0%"}`, Subs: `{"a": {"x": "xray", "y": "yankee"}, "b": [1, 2]}`, Output: `{"this": "xray", "that": 1}`, @@ -1481,7 +1479,6 @@ var unpackerTests = []UnpackerTest{ }, { Name: "ref-2", - Skip: true, Input: `{ "?": [ ["a", "b"], @@ -1496,14 +1493,12 @@ var unpackerTests = []UnpackerTest{ }, { Name: "ref-3", - Skip: true, Input: `{"a": {"b": {"c": 1}}}`, Trans: `{"a": {"rewriteValue": "%b.c"}}`, Output: `{"a": 1}`, }, { Name: "ref-4", - Skip: true, Input: `{ "a": ["alpha", "bravo", "charlie"] }`, From 80d07b74536793bf16226d790f2e54827b2f68cc Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 05:07:19 +0000 Subject: [PATCH 09/27] namespaced substitutions --- unpacker/unpacker.go | 13 ++++++------- unpacker/unpacker_test.go | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 9cd90a4..c3399a8 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -46,14 +46,17 @@ func (u *Unpacker) AddTransform(key string, trans Transform) { func (u Unpacker) Unpack(source []byte) ([]byte, error) { state := &unpackState{ctx: u, mem: make(map[string]interface{})} - // pass one, extract the question mark lookup index - if err := json.Unmarshal(source, state); err != nil { + // pass one, extract the variable-index and memoize everything for `/compact.` namespaced vars + var compact map[string]interface{} + if err := json.Unmarshal(source, &compact); err != nil { return nil, err } // memoize string replacements - augment("", state.Idx, state.mem) + augment("", compact["?"], state.mem) augment("", u.Subs, state.mem) + augment("/subs.", u.Subs, state.mem) + augment("/compact.", compact, state.mem) // fmt.Printf("Values: %#v\n", state.mem) // pass two, process the data @@ -64,7 +67,6 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { } type unpackState struct { - Idx []interface{} `json:"?"` ctx Unpacker dec *json.Decoder mem map[string]interface{} @@ -155,9 +157,6 @@ func (state unpackState) string(in string, subz map[string]interface{}) interfac } } } - - // TODO: dot lookups - // TODO: closing % replacements return in } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index fc74b37..2046f91 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1512,7 +1512,6 @@ var unpackerTests = []UnpackerTest{ }, { Name: "ref-5", - Skip: true, Input: `{"a": 1, "b": 2, "c": 3}`, Trans: `{ "a": {"rewriteValue": "%/compact.c"}, From 4bdb342eb2ebc27ceba37f77f4fcb450168ef9f9 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 14:47:38 +0000 Subject: [PATCH 10/27] parse nested transforms correctly (hopefully) --- unpacker/unpacker.go | 63 +++++++++++++++++++++++++++++++++++++-- unpacker/unpacker_test.go | 39 +++++------------------- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index c3399a8..6c757fe 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -19,13 +19,70 @@ type Transform struct { Key string `json:"rewriteKey,omitempty"` Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object + Nesting map[string]Transform `json:"-"` +} + +func (t Transform) String() string { + x, err := json.Marshal(trans(t)) + if err != nil { + return err.Error() + } + return string(x) +} + +type trans Transform + +func (t *trans) UnmarshalJSON(bits []byte) error { + x := Transform{} + dec := json.NewDecoder(bytes.NewReader(bits)) + dec.DisallowUnknownFields() + err := dec.Decode(&x) + + // attempt to recurse through nested instructions + if err != nil { + x.Nesting, err = ParseTransforms(bits) + if err != nil { + return err + } + } + *t = trans(x) + return nil +} + +// Requies some weirdness to encode the Nested definitions as well +func (t trans) MarshalJSON() ([]byte, error) { + bits, err := json.Marshal(Transform(t)) + if err != nil || len(t.Nesting) == 0 { + return bits, err + } + nest := make(map[string]trans, len(t.Nesting)) + for k, v := range t.Nesting { + nest[k] = trans(v) + } + nesting, err := json.Marshal(nest) + if err != nil { + return nil, err + } + bits = append(bits[:len(bits)-1], nesting[1:]...) + return bits, nil } type Substitution map[string]interface{} +// ParseTransforms parses a set of json tranforms. +// Transforms can be embeded, so the bulk of this logic is to properly parse embedded transforms. +// Example: {"k": {"e": {"y": {/*transform object*/}}}} func ParseTransforms(transforms []byte) (map[string]Transform, error) { - v := map[string]Transform{} - return v, json.Unmarshal(transforms, &v) + v := map[string]trans{} + err := json.Unmarshal(transforms, &v) + if err != nil { + return nil, err + } + u := make(map[string]Transform, len(v)) + for k, x := range v { + u[k] = Transform(x) + } + return u, err } func (u *Unpacker) AddSubs(subs Substitution) { @@ -171,6 +228,8 @@ func (state unpackState) delim(delim json.Delim) { } func (state unpackState) transform(dest map[string]interface{}, key string, value interface{}) { + // TODO: figure out how to respect NESTING! + trans, ok := state.ctx.Transforms[key] if !ok { dest[key] = value diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 2046f91..c03e638 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1332,42 +1332,18 @@ var unpackerTests = []UnpackerTest{ Skip: true, Input: `{ "t": 1, - "u": { - "t": 2 - }, - "v": { - "w": { - "t": 3 - } - } + "u": {"t": 2}, + "v": {"w": {"t": 3}} }`, Trans: `{ - "t": { - "rewriteKey": "testing" - }, - "u": { - "t": { - "rewriteKey": "testing2" - } - }, - "v": { - "w": { - "t": { - "rewriteKey": "testing3" - } - } - } + "t": {"rewriteKey": "testing"}, + "u": {"t": {"rewriteKey": "testing2"}}, + "v": {"w": {"t": {"rewriteKey": "testing3"}}} }`, Output: `{ "testing": 1, - "u": { - "testing2": 2 - }, - "v": { - "w": { - "testing3": 3 - } - } + "u": {"testing2": 2}, + "v": {"w": {"testing3": 3}} }`, }, { @@ -1542,6 +1518,7 @@ func (test *UnpackerTest) Run(t *testing.T) { t.Fatalf("Unable to parse transforms: %v", err) } u.Transforms = trans + // fmt.Printf("Got Transforms: %v\n", trans) } if test.Subs != `` { subs := make(Substitution) From 561febb5ec8a7c1b025d08298652ea2aa643a0cf Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 15:08:12 +0000 Subject: [PATCH 11/27] semi-proper nested transform parsing --- unpacker/unpacker.go | 35 ++++++++++++++++++++++++++++------- unpacker/unpacker_test.go | 1 - 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 6c757fe..f7f55a6 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -101,7 +101,8 @@ func (u *Unpacker) AddTransform(key string, trans Transform) { } func (u Unpacker) Unpack(source []byte) ([]byte, error) { - state := &unpackState{ctx: u, mem: make(map[string]interface{})} + state := &unpackState{mem: make(map[string]interface{})} + state.trans = append(state.trans, u.Transforms) // pass one, extract the variable-index and memoize everything for `/compact.` namespaced vars var compact map[string]interface{} @@ -124,9 +125,9 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { } type unpackState struct { - ctx Unpacker - dec *json.Decoder - mem map[string]interface{} + dec *json.Decoder + mem map[string]interface{} + trans []map[string]Transform } func (state unpackState) value() interface{} { @@ -160,11 +161,23 @@ func (state unpackState) object() map[string]interface{} { o := map[string]interface{}{} for state.dec.More() { k := state.value().(string) + + // Push transform state (for use when processing value) + trans, has_trans := state.getTransform(k) + if has_trans { + state.trans = append(state.trans, trans.Nesting) + } + v := state.value() if k == "?" { continue } state.transform(o, k, v) + + // Pop transform state + if has_trans { + state.trans = state.trans[:len(state.trans)-1] + } } state.delim('}') return o @@ -228,9 +241,7 @@ func (state unpackState) delim(delim json.Delim) { } func (state unpackState) transform(dest map[string]interface{}, key string, value interface{}) { - // TODO: figure out how to respect NESTING! - - trans, ok := state.ctx.Transforms[key] + trans, ok := state.getTransform(key) if !ok { dest[key] = value return @@ -342,3 +353,13 @@ func augment(prefix string, thing interface{}, target map[string]interface{}) { } } } + +func (state unpackState) getTransform(key string) (Transform, bool) { + for i := len(state.trans) - 1; i > 0; i-- { + if t, ok := state.trans[i][key]; ok { + return t, ok + } + } + t, ok := state.trans[0][key] + return t, ok +} diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index c03e638..2e5b738 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1329,7 +1329,6 @@ var unpackerTests = []UnpackerTest{ }, { Name: "trans-2", - Skip: true, Input: `{ "t": 1, "u": {"t": 2}, From 68637d7a7e31ea0b377986eb51c2f6ffbc9a90f2 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 15:47:33 +0000 Subject: [PATCH 12/27] getting full unpacker to work on num objects --- unpacker/unpacker.go | 36 +++-- unpacker/unpacker_test.go | 276 ++++++++------------------------------ 2 files changed, 73 insertions(+), 239 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index f7f55a6..a9798c9 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -262,13 +262,12 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu return // not much else we can do here } + // 0.5: Start building context (assign keys can modify some default values here) + ctx := map[string]interface{}{} + // 1. Assign(assignKeys) converts arrays to objects - if len(trans.Assign) != 0 { - list, ok := value.([]interface{}) - if !ok { - fmt.Println("assignKeys for " + key + " but didn't get a list") - return - } + if list, is_list := value.([]interface{}); len(trans.Assign) != 0 && is_list { + // iff value is an array, convert the keys as specified if len(list) != len(trans.Assign) { panic(fmt.Sprintf("assignKeys for %s expected %d but got %d keys", key, len(trans.Assign), len(list))) } @@ -277,19 +276,22 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu state.transform(newValue, trans.Assign[i], v) // recurse to assign key/values } value = newValue + } else if m, is_map := value.(map[string]interface{}); len(trans.Assign) != 0 && is_map { + // iff the object is a map, make sure missing keys are assigned as null (ctx resolvers use this later) + for _, k := range trans.Assign { + if _, ok := m[k]; !ok { + m[k] = nil + } + } } + // 1.5: update ctx as necessary + ctx["self"] = value + augment("", value, ctx) + // fmt.Printf("Got context: %v\n", ctx) + // 2: replacePair and exit if trans.Return != nil { - ctx := map[string]interface{}{ - "self": value, - } - if m, ok := value.(map[string]interface{}); ok { - for k, v := range m { - ctx[k] = v - } - } - // fmt.Printf("Got replacer: %v\n", value) for k, v := range trans.Return { // TODO: smarter string resolves if s, ok := v.(string); ok { @@ -302,10 +304,6 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // 3: rewriteValue if trans.Rewrite != nil { - ctx := map[string]interface{}{ - "self": value, - } - augment("", value, ctx) if m, ok := trans.Rewrite.(map[string]interface{}); ok { n := map[string]interface{}{} // ensure we don't duplicate the object for k, v := range m { diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 2e5b738..276afbc 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -132,236 +132,72 @@ var unpackerTests = []UnpackerTest{ // Source: https://www.unpacker.uk/playground?e=3 { Name: "playground-3", - Skip: true, Input: `{ "@n": 1, "o": { "n": "NUM Example Co", "s": "Example Strapline", "c": [ - { - "t": { - "d": "Customer Service", - "v": "+441270123456" - } - }, - { - "fb": { - "v": "examplefacebook" - } - }, - { - "in": { - "v": "exampleinstagram" - } - }, - { - "tw": { - "v": "exampletwitter" - } - } + {"t": {"d": "Customer Service", "v": "+441270123456"}}, + {"fb": {"v": "examplefacebook"}}, + {"in": {"v": "exampleinstagram"}}, + {"tw": {"v": "exampletwitter"}} ] } }`, Subs: `{ "locale": { - "p": { - "name": "Person" - }, - "gr": { - "name": "Group" - }, - "o": { - "name": "Organisation" - }, - "dp": { - "name": "Department" - }, - "e": { - "name": "Employee" - }, - "lc": { - "name": "Location" - }, - "gp": { - "name": "Group" - }, - "t": { - "name": "Telephone", - "default": "Call" - }, - "sm": { - "name": "SMS", - "default": "Text" - }, - "u": { - "name": "Web URL", - "default": "Click" - }, - "uu": { - "name": "Web URL (http - unsecure)", - "default": "Click" - }, - "g": { - "name": "GPS", - "default": "View Location" - }, - "a": { - "name": "Address", - "default": "View Address" - }, - "fx": { - "name": "Fax", - "default": "Send a fax" - }, - "em": { - "name": "Email", - "default": "Send an email" - }, - "aa": { - "name": "Android App", - "default": "Download the app" - }, - "as": { - "name": "iOS App", - "default": "Download the app" - }, - "bt": { - "name": "Baidu Tieba", - "default": "View Baidu profile" - }, - "fb": { - "name": "Facebook", - "default": "View Facebook profile" - }, - "fs": { - "name": "FourSquare", - "default": "View FourSquare page" - }, - "ft": { - "name": "FaceTime", - "default": "Call with Facetime" - }, - "gh": { - "name": "Github", - "default": "View Github profile" - }, - "im": { - "name": "iMessage", - "default": "Send iMessage" - }, - "in": { - "name": "Instagram", - "default": "View Instagram profile" - }, - "kk": { - "name": "Kik", - "default": "Connect with Kik" - }, - "li": { - "name": "LinkedIn", - "default": "View LinkedIn page" - }, - "ln": { - "name": "Line", - "default": "Connect with Line" - }, - "md": { - "name": "Medium", - "default": "View Medium blog" - }, - "pr": { - "name": "Periscope", - "default": "View Periscope profile" - }, - "pi": { - "name": "Pinterest", - "default": "View Pinterest board" - }, - "qq": { - "name": "QQ", - "default": "View QQ Page" - }, - "qz": { - "name": "Qzone", - "default": "View Qzone Page" - }, - "rd": { - "name": "Reddit", - "default": "View subreddit" - }, - "rn": { - "name": "Renren", - "default": "View Renren profile" - }, - "sc": { - "name": "Soundcloud", - "default": "View Soundcloud page" - }, - "sk": { - "name": "Skype", - "default": "Call with Skype" - }, - "sr": { - "name": "Swarm", - "default": "Connect with Swarm" - }, - "sn": { - "name": "Snapchat", - "default": "Connect with Snapchat" - }, - "sw": { - "name": "Sina Weibo", - "default": "View Weibo page" - }, - "tb": { - "name": "Tumblr", - "default": "View Tumblr blog" - }, - "tl": { - "name": "Telegram", - "default": "Connect with Telegram" - }, - "tw": { - "name": "Twitter", - "default": "View Twitter profile" - }, - "to": { - "name": "Twoo", - "default": "View Twoo page" - }, - "vb": { - "name": "Viber", - "default": "Call with Viber" - }, - "vk": { - "name": "Vkontakte", - "default": "View VK page" - }, - "vm": { - "name": "Vimeo", - "default": "View Vimeo profile" - }, - "wa": { - "name": "Whatsapp", - "default": "Message on Whatsapp" - }, - "wc": { - "name": "WeChat", - "default": "Connect with WeChat" - }, - "xi": { - "name": "Xing", - "default": "View Xing page" - }, - "yt": { - "name": "YouTube", - "default": "View YouTube channel" - }, - "yy": { - "name": "YY", - "default": "View YY page" - } + "p": {"name": "Person"}, + "gr": {"name": "Group"}, + "o": {"name": "Organisation"}, + "dp": {"name": "Department"}, + "e": {"name": "Employee"}, + "lc": {"name": "Location"}, + "gp": {"name": "Group"}, + "t": {"name": "Telephone", "default": "Call"}, + "sm": {"name": "SMS", "default": "Text"}, + "u": {"name": "Web URL", "default": "Click"}, + "uu": {"name": "Web URL (http - unsecure)", "default": "Click"}, + "g": {"name": "GPS", "default": "View Location"}, + "a": {"name": "Address", "default": "View Address"}, + "fx": {"name": "Fax", "default": "Send a fax"}, + "em": {"name": "Email", "default": "Send an email"}, + "aa": {"name": "Android App", "default": "Download the app"}, + "as": {"name": "iOS App", "default": "Download the app"}, + "bt": {"name": "Baidu Tieba", "default": "View Baidu profile"}, + "fb": {"name": "Facebook", "default": "View Facebook profile"}, + "fs": {"name": "FourSquare", "default": "View FourSquare page"}, + "ft": {"name": "FaceTime", "default": "Call with Facetime"}, + "gh": {"name": "Github", "default": "View Github profile"}, + "im": {"name": "iMessage", "default": "Send iMessage"}, + "in": {"name": "Instagram", "default": "View Instagram profile"}, + "kk": {"name": "Kik", "default": "Connect with Kik"}, + "li": {"name": "LinkedIn", "default": "View LinkedIn page"}, + "ln": {"name": "Line", "default": "Connect with Line"}, + "md": {"name": "Medium", "default": "View Medium blog"}, + "pr": {"name": "Periscope", "default": "View Periscope profile"}, + "pi": {"name": "Pinterest", "default": "View Pinterest board"}, + "qq": {"name": "QQ", "default": "View QQ Page"}, + "qz": {"name": "Qzone", "default": "View Qzone Page"}, + "rd": {"name": "Reddit", "default": "View subreddit"}, + "rn": {"name": "Renren", "default": "View Renren profile"}, + "sc": {"name": "Soundcloud", "default": "View Soundcloud page"}, + "sk": {"name": "Skype", "default": "Call with Skype"}, + "sr": {"name": "Swarm", "default": "Connect with Swarm"}, + "sn": {"name": "Snapchat", "default": "Connect with Snapchat"}, + "sw": {"name": "Sina Weibo", "default": "View Weibo page"}, + "tb": {"name": "Tumblr", "default": "View Tumblr blog"}, + "tl": {"name": "Telegram", "default": "Connect with Telegram"}, + "tw": {"name": "Twitter", "default": "View Twitter profile"}, + "to": {"name": "Twoo", "default": "View Twoo page"}, + "vb": {"name": "Viber", "default": "Call with Viber"}, + "vk": {"name": "Vkontakte", "default": "View VK page"}, + "vm": {"name": "Vimeo", "default": "View Vimeo profile"}, + "wa": {"name": "Whatsapp", "default": "Message on Whatsapp"}, + "wc": {"name": "WeChat", "default": "Connect with WeChat"}, + "xi": {"name": "Xing", "default": "View Xing page"}, + "yt": {"name": "YouTube", "default": "View YouTube channel"}, + "yy": {"name": "YY", "default": "View YY page"} }, "AC": "Accounts", "CS": "Customer Service" @@ -1542,7 +1378,7 @@ func (test UnpackerTest) out() []byte { if err != nil { panic(err) } - bits, err := json.Marshal(obj) + bits, err := json.MarshalIndent(obj, "", " ") if err != nil { panic(err) } From f3ac6881f9e8596918decbc9f3fb88c45d0202c5 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 22 May 2021 16:00:31 +0000 Subject: [PATCH 13/27] add docstrings --- unpacker/unpacker.go | 15 +++++++++++++++ unpacker/unpacker_test.go | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index a9798c9..38e16fc 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -1,3 +1,4 @@ +// Package unpacker implements the logic described at https://www.unpacker.uk/ to unpack compressed JSON objects. package unpacker import ( @@ -8,11 +9,17 @@ import ( "strings" ) +// Unpakcker holds a set of Transforms and Substitutions needed to unpack compressed JSON. +// This structure should allow the the Transform and Substitution object to only need to be parsed once, +// while allowing multiple unpackings using the same configuration. type Unpacker struct { Transforms map[string]Transform Subs Substitution } +// A Transform holds information on how to modify various JSON objects. +// +// docs: https://www.unpacker.uk/specification#transformation-object type Transform struct { Assign []string `json:"assignKeys,omitempty"` Items string `json:"arrayItems,omitempty"` @@ -30,6 +37,7 @@ func (t Transform) String() string { return string(x) } +// internal for now but could be exposed, makes it so we can parse nested transforms without infinite recursive loops type trans Transform func (t *trans) UnmarshalJSON(bits []byte) error { @@ -67,6 +75,9 @@ func (t trans) MarshalJSON() ([]byte, error) { return bits, nil } +// A Substitution object provides a method of removing repeditive data by having shortened keys. +// +// docs: https://www.unpacker.uk/specification#substitution-object type Substitution map[string]interface{} // ParseTransforms parses a set of json tranforms. @@ -100,6 +111,10 @@ func (u *Unpacker) AddTransform(key string, trans Transform) { u.Transforms[key] = trans } +// Unpack expands JSON using the transforms and substitutions defined on the unpacker object. +// Additionally, a viariadic index in the source data is also used to reduce size. +// +// docs: https://www.unpacker.uk/specification func (u Unpacker) Unpack(source []byte) ([]byte, error) { state := &unpackState{mem: make(map[string]interface{})} state.trans = append(state.trans, u.Transforms) diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 276afbc..3a03dd6 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -1378,7 +1378,7 @@ func (test UnpackerTest) out() []byte { if err != nil { panic(err) } - bits, err := json.MarshalIndent(obj, "", " ") + bits, err := json.Marshal(obj) if err != nil { panic(err) } From 17e7d358d03ab89f9cb1e570052e0c03c7d8c071 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sun, 23 May 2021 03:04:54 +0000 Subject: [PATCH 14/27] pull in stome initial tests from NUMtechnology/typescript-object-unpacker --- unpacker/tests.json | 1425 +++++++++++++++++++++++++++++++++++++ unpacker/unpacker.go | 13 +- unpacker/unpacker_test.go | 1366 +---------------------------------- 3 files changed, 1466 insertions(+), 1338 deletions(-) create mode 100644 unpacker/tests.json diff --git a/unpacker/tests.json b/unpacker/tests.json new file mode 100644 index 0000000..0956515 --- /dev/null +++ b/unpacker/tests.json @@ -0,0 +1,1425 @@ +[ + { + "id": "Unpacker-01", + "desc": "Arrays without transform should remain as arrays", + "input": ["alpha", "bravo", "charlie"], + "output": ["alpha", "bravo", "charlie"] + }, { + "id": "Unpacker-02", + "desc": "It should retain pairs that aren't transformed", + "input": {"a": "alpha", "b": "bravo"}, + "trans": {"b": {"rewriteValue": "bacon"}}, + "output": {"a": "alpha", "b": "bacon"} + }, { + "id": "Unpacker-03", + "desc": "When a key references an object that doesn't exist it should print null", + "input": {"a": "%a"}, + "output": {"a": null} + }, { + "id": "Unpacker-04", + "desc": "It should print references in an array at the root", + "input": ["%0"], + "output": [null] + }, { + "id": "Unpacker-05", + "desc": "It should retain map structure when referencing an object that doesn't exist", + "input": {"a": "%0"}, + "output": {"a": null} + }, { + "id": "Unpacker-06", + "desc": "You should be able to rewrite values with pass throughs", + "input": {"a": {"b": {"c": {"test": 1}}}}, + "trans": {"a": {"rewriteValue": "%/compact.a.b"}}, + "output": {"a": {"c": {"test": 1}}} + }, { + "id": "Unpacker-07", + "desc": "It should not reference the value of the first non-? key by char position", + "input": {"?":["yo","mtv","raps"],"a":"alpha"}, + "trans": {"a":{"rewriteValue":{"that":"%0"}}}, + "subs": {"a":"ASåÍa"}, + "output": {"a":{"that":null}} + }, { + "id": "Unpacker-08", + "desc": "It should retain pairs specified before the pair being replaced", + "input": {"a":"alpha","b":"bravo","c":"charlie"}, + "trans": {"b":{"replacePair":{"b1":"bravo","b2":"bacon"}}}, + "output": {"a":"alpha","b1":"bravo","b2":"bacon","c":"charlie"} + }, { + "id": "Unpacker-09", + "desc": "It should remove the pair when the value of replacePair is null", + "input": {"a":"alpha","b":"bravo","c":"charlie"}, + "trans": {"b":{"replacePair":null}}, + "output": {"a":"alpha","c":"charlie"} + }, { + "id": "Unpacker-10", + "desc": "It should be possible to rewriteValue as null", + "input": {"a":"alpha","b":"bravo","c":"charlie"}, + "trans": {"b":{"rewriteValue":null}}, + "output": {"a":"alpha","b":null,"c":"charlie"} + }, { + "id": "Unpacker-11", + "desc": "It should be possible to rewriteValue to a map with a pair with a null value", + "input": {"a":"alpha","b":"bravo","c":"charlie"}, + "trans": {"b":{"rewriteValue":{"a":null}}}, + "output": {"a":"alpha","b":{"a":null},"c":"charlie"} + }, { + "id": "Unpacker-12", + "desc": "It should be possible to replacePair with a pair with a value of null", + "input": {"a":"alpha","b":"bravo","c":"charlie"}, + "trans": {"b":{"replacePair":{"test":null}}}, + "output": {"a":"alpha","test":null,"c":"charlie"} + }, { + "id": "Unpacker-13", + "desc": "Labels shoudld not clash - ch10991", + "input": {"?":["abccompany","+4412345678"],"n":"ABC Company Ltd","t":[{"l":"Accounts","n":"%1%90"},{"l":"Customer Service","n":"%1%89"}],"tw":"/%0","i":"/%0%pics"}, + "trans": {"n":{"rewriteKey":"name"},"t":{"rewriteKey":"telephone","arrayItems":"tel"},"tel":{"l":{"rewriteKey":"label"},"n":{"rewriteKey":"number"}},"tw":{"rewriteKey":"twitter"},"i":{"rewriteKey":"instagram"}}, + "subs": {"ac":"Accounts","cs":"Customer Service"}, + "output": {"name":"ABC Company Ltd","telephone":[{"tel":{"label":"Accounts","number":"+441234567890"}},{"tel":{"label":"Customer Service","number":"+441234567889"}}],"twitter":"/abccompany","instagram":"/abccompanypics"} + }, { + "id": "Unpacker-14", + "desc": "Can rewrite keys with nested mappings", + "input": {"?":["abccompany","+4412345678"],"n":"ABC Company Ltd","t":[{"l":"%ac","d":"%1%90"},{"l":"%cs","d":"%1%89"}],"tw":"/%0","i":"/%0%pics"}, + "trans": {"n":{"rewriteKey":"name"},"t":{"arrayItems":"tel"},"tel":{"replacePair":"%self","l":{"rewriteKey":"label"},"d":{"rewriteKey":"digits"}},"tw":{"rewriteKey":"twitter"},"i":{"rewriteKey":"instagram"}}, + "subs": {"ac":"Accounts","cs":"Customer Service"}, + "output": {"name":"ABC Company Ltd","t":[{"label":"Accounts","digits":"+441234567890"},{"label":"Customer Service","digits":"+441234567889"}],"twitter":"/abccompany","instagram":"/abccompanypics"} + }, { + "id": "Unpacker-15", + "desc": "ch11021", + "input": {"o":["Widget Company Ltd","Making the best widgets","https://www.widgetcompany.com",[{"n":"Jane Smith","p":"Chief Executive Officer","b":"https://www.widgetcompany.com/team/janesmith","l":"janesmith","t":"janesmithwidgets"},{"n":"John Wilson","p":"Chief Technology Officer","b":"https://www.widgetcompany.com/team/johnwilson","l":"johnwilson","t":"jono"},{"n":"Dashna Anand","p":"Chief Marketing Officer","b":"https://www.widgetcompany.com/team/dashnaanand","l":"dashnaanand","t":"dashnaanand"}]]}, + "trans": {"o":{"rewriteKey":"organisation","assignKeys":["name","strapline","website","employees"]},"n":{"rewriteKey":"name"},"p":{"rewriteKey":"position"},"b":{"rewriteKey":"bio"},"l":{"rewriteKey":"linkedin","rewriteValue":"https://www.linkedin.com/in/%self"},"t":{"rewriteKey":"twitter","rewriteValue":"https://www.twitter.com/%self"}}, + "subs": {"ac":"Accounts","cs":"Customer Service"}, + "output": {"organisation":{"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]}} + }, { + "id": "Unpacker-16", + "desc": "ch11325", + "input": {"t":"me"}, + "trans": {"t":{"rewriteKey":"twitter","rewriteValue":{"cta":"Follow on Twitter","url":"twitter.com/%self"}}}, + "output": {"twitter":{"cta":"Follow on Twitter","url":"twitter.com/me"}} + }, { + "id": "Unpacker-17", + "desc": "ch11326", + "input": {"a":{"b":{"c":1}}}, + "trans": {"a":{"rewriteValue":"%b"}}, + "output": {"a":{"c":1}} + }, { + "id": "Unpacker-18", + "desc": "ch11326", + "input": {"a":["alpha","bravo","charlie"]}, + "trans": {"a":{"assignKeys":["a","b","c"],"rewriteValue":"%c"}}, + "output": {"a":"charlie"} + }, { + "id": "Nested-01", + "desc": "It should be possible to nest unpacker specs", + "input": {"test":{"first":{"value":1},"second":{"value":2},"third":{"value":3}}}, + "trans": {"test":{"first":{"rewriteKey":"alpha","value":{"rewriteKey":"first"}},"second":{"rewriteKey":"beta","value":{"rewriteKey":"second"}},"third":{"rewriteKey":"gamma","value":{"rewriteKey":"third"}}}}, + "output": {"test":{"alpha":{"first":1},"beta":{"second":2},"gamma":{"third":3}}} + }, { + "id": "Nested-02", + "desc": "It should be possible to nest unpacker specs", + "input": {"test":{"this":1}}, + "trans": {"test":{"this":{"rewriteKey":"that"}}}, + "output": {"test":{"that":1}} + }, { + "id": "Nested-03", + "desc": "It should be possible to nest unpacker specs", + "input": {"test":{"first":{"value1":"a","value2":"b"},"second":{"value":2},"third":{"value":3}}}, + "trans": {"test":{"rewriteKey":"TEST","first":{"rewriteKey":"alpha","value1":{"rewriteKey":"v1"},"value2":{"rewriteKey":"v2"}},"second":{"rewriteKey":"beta","value":{"rewriteKey":"second"}},"third":{"rewriteKey":"gamma","value":{"rewriteKey":"third"}}}}, + "output": {"TEST":{"alpha":{"v1":"a","v2":"b"},"beta":{"second":2},"gamma":{"third":3}}} + }, { + "id": "rewriteKey", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"f": 1}, + "trans": {"f": {"rewriteKey": "foo"}}, + "output": {"foo": 1} + }, { + "id": "rewriteValue", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"foo": 1}, + "trans": {"foo": {"rewriteValue" : { "x" : "%self", "y" : 2 }}}, + "output": {"foo": {"x": 1, "y": 2}} + }, { + "id": "replacePair", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"f": 1}, + "trans": {"f": {"replacePair": {"x": "%self", "y": 2, "z": 3}}}, + "output": {"x" : 1, "y" : 2, "z" : 3} + }, { + "id": "subs", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"foo": "%var"}, + "subs": {"var": 1}, + "output": {"foo": 1} + }, { + "id": "assignKeys", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"phonetic": ["alpha", "bravo", "charlie"]}, + "trans": {"phonetic": {"assignKeys": ["a", "b", "c"]}}, + "output": {"phonetic": {"a": "alpha", "b": "bravo", "c": "charlie"}} + }, { + "id": "arrayItems", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"employees": [["Jane Smith", "CEO"], ["Dashna Anand", "CMO"]]}, + "trans": {"employees": {"arrayItems": "employee"},"employee": {"assignKeys": ["name", "position"],"replacePair": {"name": "%name", "position": "%position"}}}, + "output": {"employees": [{"name": "Jane Smith", "position": "CEO"},{"name": "Dashna Anand", "position": "CMO"}]} + }, { + "id": "noop", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": {"employees": [{"name": "Jane Smith", "position": "CEO"},{"name": "Dashna Anand", "position": "CMO"}]}, + "output": {"employees": [{"name": "Jane Smith", "position": "CEO"},{"name": "Dashna Anand", "position": "CMO"}]} + }, { + "id": "variable-index-1", + "desc": "Source: https://www.unpacker.uk/specification#variable-index", + "input": {"?": [1], "a": "%0%"}, + "output": {"a": 1} + }, { + "id": "variable-index-2", + "desc": "Source: https://www.unpacker.uk/specification#variable-index", + "input": {"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0%", "y": "%1%"}, + "output": {"x": ["a", "b"], "y": {"a": "alpha", "b": "bravo"}} + }, { + "id": "variable-index-3", + "desc": "Source: https://www.unpacker.uk/specification#variable-index", + "input": {"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0.0%", "y": "%1.a%"}, + "output": {"x": "a", "y": "alpha"} + }, { + "id": "sub-1", + "desc": "Source: https://www.unpacker.uk/specification#substitution-object", + "input": {"this": "%a%", "that": "%b%"}, + "subs": {"a": 1, "b": 2}, + "output": {"this": 1, "that": 2} + }, { + "id": "sub-2", + "desc": "Source: https://www.unpacker.uk/specification#substitution-object", + "input": {"this": "%a.x%", "that": "%b.0%"}, + "subs": {"a": {"x": "xray", "y": "yankee"}, "b": [1, 2]}, + "output": {"this": "xray", "that": 1} + }, { + "id": "trans-1", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"t": 1,"u": {"t": 2},"v": {"w": {"t": 3}}}, + "trans": {"t": {"rewriteKey": "testing"}}, + "output": {"testing": 1,"u": {"testing": 2},"v": {"w": {"testing": 3}}} + }, { + "id": "trans-2", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": { + "t": 1, + "u": {"t": 2}, + "v": {"w": {"t": 3}} + }, + "trans": { + "t": {"rewriteKey": "testing"}, + "u": {"t": {"rewriteKey": "testing2"}}, + "v": {"w": {"t": {"rewriteKey": "testing3"}}} + }, + "output": { + "testing": 1, + "u": {"testing2": 2}, + "v": {"w": {"testing3": 3}} + } + }, { + "id": "trans-3", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"t": 1}, + "trans": {"t": {"rewriteKey": "testing"}}, + "output": {"testing": 1} + }, { + "id": "trans-4", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"t": "me"}, + "trans": { + "t": { + "rewriteKey": "twitter", + "rewriteValue": "twitter.com/%self" + } + }, + "output": {"twitter": "twitter.com/me"} + }, { + "id": "trans-5", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"t": "me"}, + "trans": { + "t": { + "rewriteKey": "twitter", + "rewriteValue": { + "cta": "Follow on Twitter", + "url": "twitter.com/%self" + } + } + }, + "output": { + "twitter": { + "cta": "Follow on Twitter", + "url": "twitter.com/me" + } + } + }, { + "id": "trans-6", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"t": "me"}, + "trans": { + "t": { + "replacePair": { + "cta": "Follow on Twitter", + "url": "twitter.com/%self" + } + } + }, + "output": {"cta": "Follow on Twitter", "url": "twitter.com/me"} + }, { + "id": "trans-7", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": {"phonetic": ["alpha", "bravo", "charlie"]}, + "trans": {"phonetic": {"assignKeys": ["a", "b", "c"]}}, + "output": { + "phonetic": { + "a": "alpha", + "b": "bravo", + "c": "charlie" + } + } + }, { + "id": "trans-8", + "desc": "Source: https://www.unpacker.uk/specification#transformation-object", + "input": { + "employees": [ + ["Jane Smith", "CEO"], + ["Dashna Anand", "CMO"] + ] + }, + "trans": { + "employees": { + "arrayItems": "employee" + }, + "employee": { + "assignKeys": [ + "name", + "position" + ], + "replacePair": { + "name": "%name", + "position": "%position" + } + } + }, + "output": {"employees": [ + {"name": "Jane Smith", "position": "CEO"}, + {"name": "Dashna Anand", "position": "CMO"} + ]} + }, { + "id": "ref-1", + "desc": "Source: https://www.unpacker.uk/specification#referencing", + "input": { + "?": ["test"], + "a": "%0%", + "b": "this is a %0", + "c": "this is another %0 of referencing", + "d": "once again %0%ing it" + }, + "output": { + "a": "test", + "b": "this is a test", + "c": "this is another test of referencing", + "d": "once again testing it" + } + }, { + "id": "ref-2", + "desc": "Source: https://www.unpacker.uk/specification#referencing", + "input": { + "?": [ + ["a", "b"], + {"a": "alpha", "b": "bravo"} + ], + "x": "%0.0", + "y": "%1.a", + "z": "%letters.0" + }, + "subs": {"letters": ["a", "b"]}, + "output": {"x": "a", "y": "alpha", "z": "a"} + }, { + "id": "ref-3", + "desc": "Source: https://www.unpacker.uk/specification#referencing", + "input": {"a": {"b": {"c": 1}}}, + "trans": {"a": {"rewriteValue": "%b.c"}}, + "output": {"a": 1} + }, { + "id": "ref-4", + "desc": "Source: https://www.unpacker.uk/specification#referencing", + "input": { + "a": ["alpha", "bravo", "charlie"] + }, + "trans": { + "a": { + "assignKeys": ["a", "b", "c"], + "rewriteValue": "%c" + } + }, + "output": {"a": "charlie"} + }, { + "id": "ref-5", + "desc": "Source: https://www.unpacker.uk/specification#referencing", + "input": {"a": 1, "b": 2, "c": 3}, + "trans": { + "a": {"rewriteValue": "%/compact.c"}, + "b": {"rewriteValue": "%/subs.x"} + }, + "subs": {"x": 4}, + "output": {"a": 3, "b": 4, "c": 3} + }, { + "id": "playground-1", + "desc": "Source: https://www.unpacker.uk/playground?e=1", + "input": { + "?": ["abccompany", "+4412345678"], + "c": "ABC Company Ltd", + "t": [ + {"l": "%ac", "n": "%1%90"}, + {"l": "%cs", "n": "%1%89"} + ], + "tw": "/%0", + "i": "/%0%pics" + }, + "trans": { + "c": {"rewriteKey": "coname"}, + "t": {"rewriteKey": "telephone"}, + "l": {"rewriteKey": "label"}, + "n": {"rewriteKey": "number"}, + "tw": {"rewriteKey": "twitter"}, + "i": {"rewriteKey": "instagram"} + }, + "subs": {"cs": "Customer Service", "ac": "Accounts"}, + "output": { + "coname": "ABC Company Ltd", + "telephone": [ + {"label": "Accounts", "number": "+441234567890"}, + {"label": "Customer Service", "number": "+441234567889"} + ], + "twitter": "/abccompany", + "instagram": "/abccompanypics" + } + }, { + "id": "playground-2", + "desc": "Source: https://www.unpacker.uk/playground?e=2", + "input": { + "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], + "o": [ + "Widget Company Ltd", + "Making the best widgets", + "%0", + [ + ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], + ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], + ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] + ] + ] + }, + "trans": { + "o": { + "assignKeys": ["n", "s", "w", "e"], + "replacePair": { + "name": "%n", + "strapline": "%s", + "website": "%w", + "employees": "%e" + } + }, + "em": { + "replacePair": { + "name": "%n", + "position": "%p", + "bio": "%b", + "linkedin": "https://www.linkedin.com/in/%l", + "twitter": "https://www.twitter.com/%t" + } + } + }, + "subs": { + "ceo": "Chief Executive Officer", + "cto": "Chief Technology Officer", + "cmo": "Chief Marketing Officer" + }, + "output": { + "name": "Widget Company Ltd", + "strapline": "Making the best widgets", + "website": "https://www.widgetcompany.com", + "employees": [ + [ + "Jane Smith", + "Chief Executive Officer", + "https://www.widgetcompany.com/team/janesmith", + "janesmith", + "janesmithwidgets" + ], + [ + "John Wilson", + "Chief Technology Officer", + "https://www.widgetcompany.com/team/johnwilson", + "johnwilson", + "jono" + ], + [ + "Dashna Anand", + "Chief Marketing Officer", + "https://www.widgetcompany.com/team/dashnaanand", + "dashnaanand", + "dashnaanand" + ] + ] + } + }, { + "id": "playground-3", + "desc": "Source: https://www.unpacker.uk/playground?e=3", + "input": { + "@n": 1, + "o": { + "n": "NUM Example Co", + "s": "Example Strapline", + "c": [ + {"t": {"d": "Customer Service", "v": "+441270123456"}}, + {"fb": {"v": "examplefacebook"}}, + {"in": {"v": "exampleinstagram"}}, + {"tw": {"v": "exampletwitter"}} + ] + } + }, + "trans": { + "o": { + "rewriteKey": "organisation", + "assignKeys": [ + "n", + "s", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.o.name", + "name": "%n", + "slogan": "%s", + "contacts": "%c" + } + }, + "p": { + "rewriteKey": "person", + "assignKeys": [ + "n", + "b", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.p.name", + "name": "%n", + "bio": "%b", + "contacts": "%c" + } + }, + "e": { + "rewriteKey": "employee", + "assignKeys": [ + "n", + "r", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.e.name", + "name": "%n", + "role": "%r", + "contacts": "%c" + } + }, + "lc": { + "rewriteKey": "location", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.lc.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "gp": { + "rewriteKey": "group", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.gp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "dp": { + "rewriteKey": "department", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.dp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "a": { + "rewriteKey": "address", + "assignKeys": [ + "al", + "pz", + "co", + "d" + ], + "rewriteValue": { + "description": "%d", + "description_default": "%/subs.locale.a.default", + "lines": "%al", + "postcode": "%pz", + "country": "%co", + "method_type": "core", + "object_display_name": "%/subs.locale.a.name", + "prefix": "" + } + }, + "l": { + "rewriteKey": "link", + "assignKeys": [ + "@L", + "d" + ], + "rewriteValue": { + "@L": "%@L", + "description": "%d" + } + }, + "fb": { + "rewriteKey": "facebook", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fb.name", + "description_default": "%/subs.locale.fb.default", + "description": "%d", + "prefix": "https://www.facebook.com/", + "method_type": "third_party", + "controller": "facebook.com", + "value": "%v" + } + }, + "g": { + "rewriteKey": "gps", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.g.name", + "description_default": "%/subs.locale.g.default", + "description": "%d", + "prefix": "", + "method_type": "core", + "value": "%v" + } + }, + "in": { + "rewriteKey": "instagram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.in.name", + "description_default": "%/subs.locale.in.default", + "description": "%d", + "prefix": "https://www.instagram.com/", + "method_type": "third_party", + "controller": "instagram.com", + "value": "%v" + } + }, + "li": { + "rewriteKey": "linkedin", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.li.name", + "description_default": "%/subs.locale.li.default", + "description": "%d", + "prefix": "https://www.linkedin.com/", + "method_type": "third_party", + "controller": "linkedin.com", + "value": "%v" + } + }, + "yt": { + "rewriteKey": "youtube", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.yt.name", + "description_default": "%/subs.locale.yt.default", + "description": "%d", + "prefix": "https://www.youtube.com/", + "method_type": "third_party", + "controller": "youtube.com", + "value": "%v" + } + }, + "pi": { + "rewriteKey": "pinterest", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.pi.name", + "description_default": "%/subs.locale.pi.default", + "description": "%d", + "prefix": "https://www.pinterest.com/", + "method_type": "third_party", + "controller": "pinterest.com", + "value": "%v" + } + }, + "tw": { + "rewriteKey": "twitter", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.tw.name", + "description_default": "%/subs.locale.tw.default", + "description": "%d", + "prefix": "https://www.twitter.com/", + "method_type": "third_party", + "controller": "twitter.com", + "value": "%v", + "value_prefix": "@" + } + }, + "t": { + "rewriteKey": "telephone", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.t.name", + "description_default": "%/subs.locale.t.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "sm": { + "rewriteKey": "sms", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.sm.name", + "description_default": "%/subs.locale.sm.default", + "description": "%d", + "prefix": "sms:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "em": { + "rewriteKey": "email", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.em.name", + "description_default": "%/subs.locale.em.default", + "description": "%d", + "prefix": "mailto:", + "method_type": "core", + "value": "%v" + } + }, + "fx": { + "rewriteKey": "fax", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fx.name", + "description_default": "%/subs.locale.fx.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v" + } + }, + "u": { + "rewriteKey": "url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.u.name", + "description_default": "%/subs.locale.u.default", + "description": "%d", + "prefix": "https://", + "method_type": "core", + "value": "%v" + } + }, + "uu": { + "rewriteKey": "unsecure_url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.uu.name", + "description_default": "%/subs.locale.uu.default", + "description": "%d", + "prefix": "http://", + "method_type": "core", + "value": "%v" + } + }, + "av": { + "rewriteKey": "available" + }, + "tz": { + "rewriteKey": "time_zone_location" + }, + "i": { + "rewriteKey": "introduction" + }, + "ac": { + "rewriteKey": "access" + }, + "aa": { + "rewriteKey": "android-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.aa.name", + "description_default": "locale.aa.default", + "prefix": "https://play.google.com/store/apps/details?id=", + "method_type": "third_party", + "controller": "play.google.com" + } + }, + "as": { + "rewriteKey": "ios-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.as.name", + "description_default": "locale.as.default", + "prefix": "https://itunes.apple.com/app/", + "method_type": "third_party", + "controller": "apps.apple.com" + } + }, + "bt": { + "rewriteKey": "baidu_tieba", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.bt.name", + "description_default": "locale.bt.default", + "prefix": "https://tieba.baidu.com/", + "method_type": "third_party", + "controller": "tieba.baidu.com" + } + }, + "fs": { + "rewriteKey": "foursquare", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.fs.name", + "description_default": "locale.fs.default", + "prefix": "https://www.foursquare.com/", + "method_type": "third_party", + "controller": "foursquare.com" + } + }, + "ft": { + "rewriteKey": "facetime", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ft.name", + "description_default": "locale.ft.default", + "prefix": "facetime://", + "method_type": "third_party", + "controller": "facetime@apple.com" + } + }, + "gh": { + "rewriteKey": "github", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.gh.name", + "description_default": "locale.gh.default", + "prefix": "https://www.github.com/", + "method_type": "third_party", + "controller": "github.com" + } + }, + "im": { + "rewriteKey": "imessage", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.im.name", + "description_default": "locale.im.default", + "prefix": "imessage://", + "method_type": "third_party", + "controller": "imessage@apple.com" + } + }, + "kk": { + "rewriteKey": "kik", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.kk.name", + "description_default": "locale.kk.default", + "prefix": "https://www.kik.com/u/", + "method_type": "third_party", + "controller": "kik.com" + } + }, + "ln": { + "rewriteKey": "line", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ln.name", + "description_default": "locale.ln.default", + "prefix": "line://", + "method_type": "third_party", + "controller": "line.me" + } + }, + "md": { + "rewriteKey": "medium", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.md.name", + "description_default": "locale.md.default", + "prefix": "https://www.medium.com/", + "method_type": "third_party", + "controller": "medium.com" + } + }, + "pr": { + "rewriteKey": "periscope", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.pr.name", + "description_default": "locale.pr.default", + "prefix": "https://www.periscope.tv/", + "method_type": "third_party", + "controller": "periscope.tv" + } + }, + "qq": { + "rewriteKey": "qq", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qq.name", + "description_default": "locale.qq.default", + "prefix": "https://www.qq.com/", + "method_type": "third_party", + "controller": "qq.com" + } + }, + "qz": { + "rewriteKey": "qzone", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qz.name", + "description_default": "locale.qz.default", + "prefix": "https://www.qzone.com/", + "method_type": "third_party", + "controller": "qzone.com" + } + }, + "rd": { + "rewriteKey": "reddit", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rd.name", + "description_default": "locale.rd.default", + "prefix": "https://www.reddit.com/r/", + "method_type": "third_party", + "controller": "reddit.com" + } + }, + "rn": { + "rewriteKey": "renren", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rn.name", + "description_default": "locale.rn.default", + "prefix": "https://www.renren.com/", + "method_type": "third_party", + "controller": "renren.com" + } + }, + "sc": { + "rewriteKey": "soundcloud", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sc.name", + "description_default": "locale.sc.default", + "prefix": "https://www.soundcloud.com/", + "method_type": "third_party", + "controller": "soundcloud.com" + } + }, + "sk": { + "rewriteKey": "skype", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sk.name", + "description_default": "locale.sk.default", + "prefix": "skype:", + "method_type": "third_party", + "controller": "skype.com" + } + }, + "sr": { + "rewriteKey": "swarm", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sr.name", + "description_default": "locale.sr.default", + "prefix": "https://www.swarmapp.com/", + "method_type": "third_party", + "controller": "swarmapp.com" + } + }, + "sn": { + "rewriteKey": "snapchat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sn.name", + "description_default": "locale.sn.default", + "prefix": "snapchat://add/", + "method_type": "third_party", + "controller": "snapchat.com" + } + }, + "sw": { + "rewriteKey": "sina-weibo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sw.name", + "description_default": "locale.sw.default", + "prefix": "https://www.weibo.com/", + "method_type": "third_party", + "controller": "weibo.com" + } + }, + "tb": { + "rewriteKey": "tumblr", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tb.name", + "description_default": "locale.tb.default", + "prefix": "https://.tumblr.com/", + "method_type": "third_party", + "controller": "tumblr.com" + } + }, + "tl": { + "rewriteKey": "telegram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tl.name", + "description_default": "locale.tl.default", + "prefix": "https://www.telegram.me/", + "method_type": "third_party", + "controller": "telegram.com" + } + }, + "to": { + "rewriteKey": "twoo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.to.name", + "description_default": "locale.to.default", + "prefix": "https://www.twoo.com/", + "method_type": "third_party", + "controller": "twoo.com" + } + }, + "vb": { + "rewriteKey": "viber", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vb.name", + "description_default": "locale.vb.default", + "prefix": "https://www.viber.com/", + "method_type": "third_party", + "controller": "viber.com" + } + }, + "vk": { + "rewriteKey": "vkontakte", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vk.name", + "description_default": "locale.vk.default", + "prefix": "https://www.vk.com/", + "method_type": "third_party", + "controller": "vk.com" + } + }, + "vm": { + "rewriteKey": "vimeo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vm.name", + "description_default": "locale.vm.default", + "prefix": "https://www.vimeo.com/", + "method_type": "third_party", + "controller": "vimeo.com" + } + }, + "wa": { + "rewriteKey": "whatsapp", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wa.name", + "description_default": "locale.wa.default", + "prefix": "whatsapp://", + "method_type": "third_party", + "controller": "whatsapp.com" + } + }, + "wc": { + "rewriteKey": "wechat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wc.name", + "description_default": "locale.wc.default", + "prefix": "https://www.wechat.com/", + "method_type": "third_party", + "controller": "wechat.com" + } + }, + "xi": { + "rewriteKey": "xing", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.xi.name", + "description_default": "locale.xi.default", + "prefix": "https://www.xing.com/", + "method_type": "third_party", + "controller": "xing.com" + } + }, + "yy": { + "rewriteKey": "yy", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.yy.name", + "description_default": "locale.yy.default", + "prefix": "https://www.yy.com/", + "method_type": "third_party", + "controller": "yy.com" + } + } + }, + "subs": { + "locale": { + "p": {"name": "Person"}, + "gr": {"name": "Group"}, + "o": {"name": "Organisation"}, + "dp": {"name": "Department"}, + "e": {"name": "Employee"}, + "lc": {"name": "Location"}, + "gp": {"name": "Group"}, + "t": {"name": "Telephone", "default": "Call"}, + "sm": {"name": "SMS", "default": "Text"}, + "u": {"name": "Web URL", "default": "Click"}, + "uu": {"name": "Web URL (http - unsecure)", "default": "Click"}, + "g": {"name": "GPS", "default": "View Location"}, + "a": {"name": "Address", "default": "View Address"}, + "fx": {"name": "Fax", "default": "Send a fax"}, + "em": {"name": "Email", "default": "Send an email"}, + "aa": {"name": "Android App", "default": "Download the app"}, + "as": {"name": "iOS App", "default": "Download the app"}, + "bt": {"name": "Baidu Tieba", "default": "View Baidu profile"}, + "fb": {"name": "Facebook", "default": "View Facebook profile"}, + "fs": {"name": "FourSquare", "default": "View FourSquare page"}, + "ft": {"name": "FaceTime", "default": "Call with Facetime"}, + "gh": {"name": "Github", "default": "View Github profile"}, + "im": {"name": "iMessage", "default": "Send iMessage"}, + "in": {"name": "Instagram", "default": "View Instagram profile"}, + "kk": {"name": "Kik", "default": "Connect with Kik"}, + "li": {"name": "LinkedIn", "default": "View LinkedIn page"}, + "ln": {"name": "Line", "default": "Connect with Line"}, + "md": {"name": "Medium", "default": "View Medium blog"}, + "pr": {"name": "Periscope", "default": "View Periscope profile"}, + "pi": {"name": "Pinterest", "default": "View Pinterest board"}, + "qq": {"name": "QQ", "default": "View QQ Page"}, + "qz": {"name": "Qzone", "default": "View Qzone Page"}, + "rd": {"name": "Reddit", "default": "View subreddit"}, + "rn": {"name": "Renren", "default": "View Renren profile"}, + "sc": {"name": "Soundcloud", "default": "View Soundcloud page"}, + "sk": {"name": "Skype", "default": "Call with Skype"}, + "sr": {"name": "Swarm", "default": "Connect with Swarm"}, + "sn": {"name": "Snapchat", "default": "Connect with Snapchat"}, + "sw": {"name": "Sina Weibo", "default": "View Weibo page"}, + "tb": {"name": "Tumblr", "default": "View Tumblr blog"}, + "tl": {"name": "Telegram", "default": "Connect with Telegram"}, + "tw": {"name": "Twitter", "default": "View Twitter profile"}, + "to": {"name": "Twoo", "default": "View Twoo page"}, + "vb": {"name": "Viber", "default": "Call with Viber"}, + "vk": {"name": "Vkontakte", "default": "View VK page"}, + "vm": {"name": "Vimeo", "default": "View Vimeo profile"}, + "wa": {"name": "Whatsapp", "default": "Message on Whatsapp"}, + "wc": {"name": "WeChat", "default": "Connect with WeChat"}, + "xi": {"name": "Xing", "default": "View Xing page"}, + "yt": {"name": "YouTube", "default": "View YouTube channel"}, + "yy": {"name": "YY", "default": "View YY page"} + }, + "AC": "Accounts", + "CS": "Customer Service" + }, + "output": { + "@n": 1, + "organisation": { + "object_display_name": "Organisation", + "name": "NUM Example Co", + "slogan": "Example Strapline", + "contacts": [ + { + "telephone": { + "object_display_name": "Telephone", + "description_default": "Call", + "description": "Customer Service", + "prefix": "tel:", + "method_type": "core", + "value": "+441270123456", + "hours": null + } + }, + { + "facebook": { + "object_display_name": "Facebook", + "description_default": "View Facebook profile", + "description": null, + "prefix": "https://www.facebook.com/", + "method_type": "third_party", + "controller": "facebook.com", + "value": "examplefacebook" + } + }, + { + "instagram": { + "object_display_name": "Instagram", + "description_default": "View Instagram profile", + "description": null, + "prefix": "https://www.instagram.com/", + "method_type": "third_party", + "controller": "instagram.com", + "value": "exampleinstagram" + } + }, + { + "twitter": { + "object_display_name": "Twitter", + "description_default": "View Twitter profile", + "description": null, + "prefix": "https://www.twitter.com/", + "method_type": "third_party", + "controller": "twitter.com", + "value": "exampletwitter", + "value_prefix": "@" + } + } + ] + } + } + }, { + "id": "all", + "desc": "Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d", + "input": { + "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], + "o": [ + "Widget Company Ltd", + "Making the best widgets", + "%0", + [ + ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], + ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], + ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] + ] + ] + }, + "trans": { + "o": { + "assignKeys": ["n", "s", "w", "e"], + "replacePair": { + "name": "%n", + "strapline": "%s", + "website": "%w", + "employees": "%e" + } + }, + "e" : { + "arrayItems" : "em" + }, + "em": { + "assignKeys": ["n", "p", "b", "l", "t"], + "replacePair": { + "name": "%n", + "position": "%p", + "bio": "%b", + "linkedin": "https://www.linkedin.com/in/%l", + "twitter": "https://www.twitter.com/%t" + } + } + }, + "subs": { + "ceo": "Chief Executive Officer", + "cto": "Chief Technology Officer", + "cmo": "Chief Marketing Officer" + }, + "output": { + "name": "Widget Company Ltd", + "strapline": "Making the best widgets", + "website": "https://www.widgetcompany.com", + "employees": [{ + "name": "Jane Smith", + "position": "Chief Executive Officer", + "bio": "https://www.widgetcompany.com/team/janesmith", + "linkedin": "https://www.linkedin.com/in/janesmith", + "twitter": "https://www.twitter.com/janesmithwidgets" + }, { + "name": "John Wilson", + "position": "Chief Technology Officer", + "bio": "https://www.widgetcompany.com/team/johnwilson", + "linkedin": "https://www.linkedin.com/in/johnwilson", + "twitter": "https://www.twitter.com/jono" + }, { + "name": "Dashna Anand", + "position": "Chief Marketing Officer", + "bio": "https://www.widgetcompany.com/team/dashnaanand", + "linkedin": "https://www.linkedin.com/in/dashnaanand", + "twitter": "https://www.twitter.com/dashnaanand" + }] + } + }, { + "id": "Unpacker-", + "desc": "", + "input": null, + "output": null + } +] \ No newline at end of file diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 38e16fc..1ab5e44 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -87,6 +87,7 @@ func ParseTransforms(transforms []byte) (map[string]Transform, error) { v := map[string]trans{} err := json.Unmarshal(transforms, &v) if err != nil { + fmt.Printf("failed parsing: %s\n", string(transforms)) return nil, err } u := make(map[string]Transform, len(v)) @@ -120,13 +121,15 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { state.trans = append(state.trans, u.Transforms) // pass one, extract the variable-index and memoize everything for `/compact.` namespaced vars - var compact map[string]interface{} + var compact interface{} if err := json.Unmarshal(source, &compact); err != nil { return nil, err } // memoize string replacements - augment("", compact["?"], state.mem) + if obj, ok := compact.(map[string]interface{}); ok { + augment("", obj["?"], state.mem) + } augment("", u.Subs, state.mem) augment("/subs.", u.Subs, state.mem) augment("/compact.", compact, state.mem) @@ -167,6 +170,8 @@ func (state unpackState) value() interface{} { case float64: // fmt.Printf("Got an int value: %f\n", token) return v + case nil: + return nil default: panic(fmt.Sprintf("Unknown type: %T", token)) } @@ -242,6 +247,10 @@ func (state unpackState) string(in string, subz map[string]interface{}) interfac } } } + // assume some part of the full thing was a key (likely a bug, need a beter way to do this) + if strings.HasPrefix(in, "%") { + return nil + } return in } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 3a03dd6..34a3b1e 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -3,1342 +3,16 @@ package unpacker import ( "bytes" "encoding/json" + "io/ioutil" "testing" ) -var unpackerTests = []UnpackerTest{ - // Source: https://www.notion.so/num/Unpacker-Specification-f3385257766e44b4abf70fec4650ff7d - { - Name: "all", - Input: `{ - "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], - "o": [ - "Widget Company Ltd", - "Making the best widgets", - "%0", - [ - ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], - ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], - ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] - ] - ] - }`, - Subs: `{ - "ceo": "Chief Executive Officer", - "cto": "Chief Technology Officer", - "cmo": "Chief Marketing Officer" - }`, - Trans: `{ - "o": { - "assignKeys": ["n", "s", "w", "e"], - "replacePair": { - "name": "%n", - "strapline": "%s", - "website": "%w", - "employees": "%e" - } - }, - "e" : { - "arrayItems" : "em" - }, - "em": { - "assignKeys": ["n", "p", "b", "l", "t"], - "replacePair": { - "name": "%n", - "position": "%p", - "bio": "%b", - "linkedin": "https://www.linkedin.com/in/%l", - "twitter": "https://www.twitter.com/%t" - } - } - }`, - Output: `{ - "name": "Widget Company Ltd", - "strapline": "Making the best widgets", - "website": "https://www.widgetcompany.com", - "employees": [{ - "name": "Jane Smith", - "position": "Chief Executive Officer", - "bio": "https://www.widgetcompany.com/team/janesmith", - "linkedin": "https://www.linkedin.com/in/janesmith", - "twitter": "https://www.twitter.com/janesmithwidgets" - }, { - "name": "John Wilson", - "position": "Chief Technology Officer", - "bio": "https://www.widgetcompany.com/team/johnwilson", - "linkedin": "https://www.linkedin.com/in/johnwilson", - "twitter": "https://www.twitter.com/jono" - }, { - "name": "Dashna Anand", - "position": "Chief Marketing Officer", - "bio": "https://www.widgetcompany.com/team/dashnaanand", - "linkedin": "https://www.linkedin.com/in/dashnaanand", - "twitter": "https://www.twitter.com/dashnaanand" - }] - }`, - }, { - Name: "rewriteKey", - Input: `{"f": 1}`, - Trans: `{"f": {"rewriteKey": "foo"}}`, - Output: `{"foo": 1}`, - }, { - Name: "rewriteValue", - Input: `{"foo": 1}`, - Trans: `{"foo": {"rewriteValue" : { "x" : "%self", "y" : 2 }}}`, - Output: `{"foo": {"x": 1, "y": 2}}`, - }, { - Name: "replacePair", - Input: `{"f": 1}`, - Trans: `{"f": {"replacePair": {"x": "%self", "y": 2, "z": 3}}}`, - Output: `{"x" : 1, "y" : 2, "z" : 3}`, - }, { - Name: "subs", - Input: `{"foo": "%var"}`, - Subs: `{"var": 1}`, - Output: `{"foo": 1}`, - }, { - Name: "assignKeys", - Input: `{"phonetic": ["alpha", "bravo", "charlie"]}`, - Trans: `{"phonetic": {"assignKeys": ["a", "b", "c"]}}`, - Output: `{"phonetic": {"a": "alpha", "b": "bravo", "c": "charlie"}}`, - }, { - Name: "arrayItems", - Input: `{"employees": [["Jane Smith", "CEO"], ["Dashna Anand", "CMO"]]}`, - Trans: `{ - "employees": {"arrayItems": "employee"}, - "employee": { - "assignKeys": ["name", "position"], - "replacePair": {"name": "%name", "position": "%position"} - }}`, - Output: `{ - "employees": [ - {"name": "Jane Smith", "position": "CEO"}, - {"name": "Dashna Anand", "position": "CMO"} - ]}`, - }, { - Name: "noop", - Input: `{ - "employees": [ - {"name": "Jane Smith", "position": "CEO"}, - {"name": "Dashna Anand", "position": "CMO"} - ]}`, - Output: `{ - "employees": [ - {"name": "Jane Smith", "position": "CEO"}, - {"name": "Dashna Anand", "position": "CMO"} - ]}`, - }, - - // Source: https://www.unpacker.uk/playground?e=3 - { - Name: "playground-3", - Input: `{ - "@n": 1, - "o": { - "n": "NUM Example Co", - "s": "Example Strapline", - "c": [ - {"t": {"d": "Customer Service", "v": "+441270123456"}}, - {"fb": {"v": "examplefacebook"}}, - {"in": {"v": "exampleinstagram"}}, - {"tw": {"v": "exampletwitter"}} - ] - } - }`, - Subs: `{ - "locale": { - "p": {"name": "Person"}, - "gr": {"name": "Group"}, - "o": {"name": "Organisation"}, - "dp": {"name": "Department"}, - "e": {"name": "Employee"}, - "lc": {"name": "Location"}, - "gp": {"name": "Group"}, - "t": {"name": "Telephone", "default": "Call"}, - "sm": {"name": "SMS", "default": "Text"}, - "u": {"name": "Web URL", "default": "Click"}, - "uu": {"name": "Web URL (http - unsecure)", "default": "Click"}, - "g": {"name": "GPS", "default": "View Location"}, - "a": {"name": "Address", "default": "View Address"}, - "fx": {"name": "Fax", "default": "Send a fax"}, - "em": {"name": "Email", "default": "Send an email"}, - "aa": {"name": "Android App", "default": "Download the app"}, - "as": {"name": "iOS App", "default": "Download the app"}, - "bt": {"name": "Baidu Tieba", "default": "View Baidu profile"}, - "fb": {"name": "Facebook", "default": "View Facebook profile"}, - "fs": {"name": "FourSquare", "default": "View FourSquare page"}, - "ft": {"name": "FaceTime", "default": "Call with Facetime"}, - "gh": {"name": "Github", "default": "View Github profile"}, - "im": {"name": "iMessage", "default": "Send iMessage"}, - "in": {"name": "Instagram", "default": "View Instagram profile"}, - "kk": {"name": "Kik", "default": "Connect with Kik"}, - "li": {"name": "LinkedIn", "default": "View LinkedIn page"}, - "ln": {"name": "Line", "default": "Connect with Line"}, - "md": {"name": "Medium", "default": "View Medium blog"}, - "pr": {"name": "Periscope", "default": "View Periscope profile"}, - "pi": {"name": "Pinterest", "default": "View Pinterest board"}, - "qq": {"name": "QQ", "default": "View QQ Page"}, - "qz": {"name": "Qzone", "default": "View Qzone Page"}, - "rd": {"name": "Reddit", "default": "View subreddit"}, - "rn": {"name": "Renren", "default": "View Renren profile"}, - "sc": {"name": "Soundcloud", "default": "View Soundcloud page"}, - "sk": {"name": "Skype", "default": "Call with Skype"}, - "sr": {"name": "Swarm", "default": "Connect with Swarm"}, - "sn": {"name": "Snapchat", "default": "Connect with Snapchat"}, - "sw": {"name": "Sina Weibo", "default": "View Weibo page"}, - "tb": {"name": "Tumblr", "default": "View Tumblr blog"}, - "tl": {"name": "Telegram", "default": "Connect with Telegram"}, - "tw": {"name": "Twitter", "default": "View Twitter profile"}, - "to": {"name": "Twoo", "default": "View Twoo page"}, - "vb": {"name": "Viber", "default": "Call with Viber"}, - "vk": {"name": "Vkontakte", "default": "View VK page"}, - "vm": {"name": "Vimeo", "default": "View Vimeo profile"}, - "wa": {"name": "Whatsapp", "default": "Message on Whatsapp"}, - "wc": {"name": "WeChat", "default": "Connect with WeChat"}, - "xi": {"name": "Xing", "default": "View Xing page"}, - "yt": {"name": "YouTube", "default": "View YouTube channel"}, - "yy": {"name": "YY", "default": "View YY page"} - }, - "AC": "Accounts", - "CS": "Customer Service" - }`, - Trans: `{ - "o": { - "rewriteKey": "organisation", - "assignKeys": [ - "n", - "s", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.o.name", - "name": "%n", - "slogan": "%s", - "contacts": "%c" - } - }, - "p": { - "rewriteKey": "person", - "assignKeys": [ - "n", - "b", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.p.name", - "name": "%n", - "bio": "%b", - "contacts": "%c" - } - }, - "e": { - "rewriteKey": "employee", - "assignKeys": [ - "n", - "r", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.e.name", - "name": "%n", - "role": "%r", - "contacts": "%c" - } - }, - "lc": { - "rewriteKey": "location", - "assignKeys": [ - "n", - "d", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.lc.name", - "name": "%n", - "description": "%d", - "contacts": "%c" - } - }, - "gp": { - "rewriteKey": "group", - "assignKeys": [ - "n", - "d", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.gp.name", - "name": "%n", - "description": "%d", - "contacts": "%c" - } - }, - "dp": { - "rewriteKey": "department", - "assignKeys": [ - "n", - "d", - "c" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.dp.name", - "name": "%n", - "description": "%d", - "contacts": "%c" - } - }, - "a": { - "rewriteKey": "address", - "assignKeys": [ - "al", - "pz", - "co", - "d" - ], - "rewriteValue": { - "description": "%d", - "description_default": "%/subs.locale.a.default", - "lines": "%al", - "postcode": "%pz", - "country": "%co", - "method_type": "core", - "object_display_name": "%/subs.locale.a.name", - "prefix": "" - } - }, - "l": { - "rewriteKey": "link", - "assignKeys": [ - "@L", - "d" - ], - "rewriteValue": { - "@L": "%@L", - "description": "%d" - } - }, - "fb": { - "rewriteKey": "facebook", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.fb.name", - "description_default": "%/subs.locale.fb.default", - "description": "%d", - "prefix": "https://www.facebook.com/", - "method_type": "third_party", - "controller": "facebook.com", - "value": "%v" - } - }, - "g": { - "rewriteKey": "gps", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.g.name", - "description_default": "%/subs.locale.g.default", - "description": "%d", - "prefix": "", - "method_type": "core", - "value": "%v" - } - }, - "in": { - "rewriteKey": "instagram", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.in.name", - "description_default": "%/subs.locale.in.default", - "description": "%d", - "prefix": "https://www.instagram.com/", - "method_type": "third_party", - "controller": "instagram.com", - "value": "%v" - } - }, - "li": { - "rewriteKey": "linkedin", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.li.name", - "description_default": "%/subs.locale.li.default", - "description": "%d", - "prefix": "https://www.linkedin.com/", - "method_type": "third_party", - "controller": "linkedin.com", - "value": "%v" - } - }, - "yt": { - "rewriteKey": "youtube", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.yt.name", - "description_default": "%/subs.locale.yt.default", - "description": "%d", - "prefix": "https://www.youtube.com/", - "method_type": "third_party", - "controller": "youtube.com", - "value": "%v" - } - }, - "pi": { - "rewriteKey": "pinterest", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.pi.name", - "description_default": "%/subs.locale.pi.default", - "description": "%d", - "prefix": "https://www.pinterest.com/", - "method_type": "third_party", - "controller": "pinterest.com", - "value": "%v" - } - }, - "tw": { - "rewriteKey": "twitter", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.tw.name", - "description_default": "%/subs.locale.tw.default", - "description": "%d", - "prefix": "https://www.twitter.com/", - "method_type": "third_party", - "controller": "twitter.com", - "value": "%v", - "value_prefix": "@" - } - }, - "t": { - "rewriteKey": "telephone", - "assignKeys": [ - "v", - "d", - "h" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.t.name", - "description_default": "%/subs.locale.t.default", - "description": "%d", - "prefix": "tel:", - "method_type": "core", - "value": "%v", - "hours": "%h" - } - }, - "sm": { - "rewriteKey": "sms", - "assignKeys": [ - "v", - "d", - "h" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.sm.name", - "description_default": "%/subs.locale.sm.default", - "description": "%d", - "prefix": "sms:", - "method_type": "core", - "value": "%v", - "hours": "%h" - } - }, - "em": { - "rewriteKey": "email", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.em.name", - "description_default": "%/subs.locale.em.default", - "description": "%d", - "prefix": "mailto:", - "method_type": "core", - "value": "%v" - } - }, - "fx": { - "rewriteKey": "fax", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.fx.name", - "description_default": "%/subs.locale.fx.default", - "description": "%d", - "prefix": "tel:", - "method_type": "core", - "value": "%v" - } - }, - "u": { - "rewriteKey": "url", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.u.name", - "description_default": "%/subs.locale.u.default", - "description": "%d", - "prefix": "https://", - "method_type": "core", - "value": "%v" - } - }, - "uu": { - "rewriteKey": "unsecure_url", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "%/subs.locale.uu.name", - "description_default": "%/subs.locale.uu.default", - "description": "%d", - "prefix": "http://", - "method_type": "core", - "value": "%v" - } - }, - "av": { - "rewriteKey": "available" - }, - "tz": { - "rewriteKey": "time_zone_location" - }, - "i": { - "rewriteKey": "introduction" - }, - "ac": { - "rewriteKey": "access" - }, - "aa": { - "rewriteKey": "android-app", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.aa.name", - "description_default": "locale.aa.default", - "prefix": "https://play.google.com/store/apps/details?id=", - "method_type": "third_party", - "controller": "play.google.com" - } - }, - "as": { - "rewriteKey": "ios-app", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.as.name", - "description_default": "locale.as.default", - "prefix": "https://itunes.apple.com/app/", - "method_type": "third_party", - "controller": "apps.apple.com" - } - }, - "bt": { - "rewriteKey": "baidu_tieba", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.bt.name", - "description_default": "locale.bt.default", - "prefix": "https://tieba.baidu.com/", - "method_type": "third_party", - "controller": "tieba.baidu.com" - } - }, - "fs": { - "rewriteKey": "foursquare", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.fs.name", - "description_default": "locale.fs.default", - "prefix": "https://www.foursquare.com/", - "method_type": "third_party", - "controller": "foursquare.com" - } - }, - "ft": { - "rewriteKey": "facetime", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.ft.name", - "description_default": "locale.ft.default", - "prefix": "facetime://", - "method_type": "third_party", - "controller": "facetime@apple.com" - } - }, - "gh": { - "rewriteKey": "github", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.gh.name", - "description_default": "locale.gh.default", - "prefix": "https://www.github.com/", - "method_type": "third_party", - "controller": "github.com" - } - }, - "im": { - "rewriteKey": "imessage", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.im.name", - "description_default": "locale.im.default", - "prefix": "imessage://", - "method_type": "third_party", - "controller": "imessage@apple.com" - } - }, - "kk": { - "rewriteKey": "kik", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.kk.name", - "description_default": "locale.kk.default", - "prefix": "https://www.kik.com/u/", - "method_type": "third_party", - "controller": "kik.com" - } - }, - "ln": { - "rewriteKey": "line", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.ln.name", - "description_default": "locale.ln.default", - "prefix": "line://", - "method_type": "third_party", - "controller": "line.me" - } - }, - "md": { - "rewriteKey": "medium", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.md.name", - "description_default": "locale.md.default", - "prefix": "https://www.medium.com/", - "method_type": "third_party", - "controller": "medium.com" - } - }, - "pr": { - "rewriteKey": "periscope", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.pr.name", - "description_default": "locale.pr.default", - "prefix": "https://www.periscope.tv/", - "method_type": "third_party", - "controller": "periscope.tv" - } - }, - "qq": { - "rewriteKey": "qq", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.qq.name", - "description_default": "locale.qq.default", - "prefix": "https://www.qq.com/", - "method_type": "third_party", - "controller": "qq.com" - } - }, - "qz": { - "rewriteKey": "qzone", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.qz.name", - "description_default": "locale.qz.default", - "prefix": "https://www.qzone.com/", - "method_type": "third_party", - "controller": "qzone.com" - } - }, - "rd": { - "rewriteKey": "reddit", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.rd.name", - "description_default": "locale.rd.default", - "prefix": "https://www.reddit.com/r/", - "method_type": "third_party", - "controller": "reddit.com" - } - }, - "rn": { - "rewriteKey": "renren", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.rn.name", - "description_default": "locale.rn.default", - "prefix": "https://www.renren.com/", - "method_type": "third_party", - "controller": "renren.com" - } - }, - "sc": { - "rewriteKey": "soundcloud", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.sc.name", - "description_default": "locale.sc.default", - "prefix": "https://www.soundcloud.com/", - "method_type": "third_party", - "controller": "soundcloud.com" - } - }, - "sk": { - "rewriteKey": "skype", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.sk.name", - "description_default": "locale.sk.default", - "prefix": "skype:", - "method_type": "third_party", - "controller": "skype.com" - } - }, - "sr": { - "rewriteKey": "swarm", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.sr.name", - "description_default": "locale.sr.default", - "prefix": "https://www.swarmapp.com/", - "method_type": "third_party", - "controller": "swarmapp.com" - } - }, - "sn": { - "rewriteKey": "snapchat", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.sn.name", - "description_default": "locale.sn.default", - "prefix": "snapchat://add/", - "method_type": "third_party", - "controller": "snapchat.com" - } - }, - "sw": { - "rewriteKey": "sina-weibo", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.sw.name", - "description_default": "locale.sw.default", - "prefix": "https://www.weibo.com/", - "method_type": "third_party", - "controller": "weibo.com" - } - }, - "tb": { - "rewriteKey": "tumblr", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.tb.name", - "description_default": "locale.tb.default", - "prefix": "https://.tumblr.com/", - "method_type": "third_party", - "controller": "tumblr.com" - } - }, - "tl": { - "rewriteKey": "telegram", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.tl.name", - "description_default": "locale.tl.default", - "prefix": "https://www.telegram.me/", - "method_type": "third_party", - "controller": "telegram.com" - } - }, - "to": { - "rewriteKey": "twoo", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.to.name", - "description_default": "locale.to.default", - "prefix": "https://www.twoo.com/", - "method_type": "third_party", - "controller": "twoo.com" - } - }, - "vb": { - "rewriteKey": "viber", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.vb.name", - "description_default": "locale.vb.default", - "prefix": "https://www.viber.com/", - "method_type": "third_party", - "controller": "viber.com" - } - }, - "vk": { - "rewriteKey": "vkontakte", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.vk.name", - "description_default": "locale.vk.default", - "prefix": "https://www.vk.com/", - "method_type": "third_party", - "controller": "vk.com" - } - }, - "vm": { - "rewriteKey": "vimeo", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.vm.name", - "description_default": "locale.vm.default", - "prefix": "https://www.vimeo.com/", - "method_type": "third_party", - "controller": "vimeo.com" - } - }, - "wa": { - "rewriteKey": "whatsapp", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.wa.name", - "description_default": "locale.wa.default", - "prefix": "whatsapp://", - "method_type": "third_party", - "controller": "whatsapp.com" - } - }, - "wc": { - "rewriteKey": "wechat", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.wc.name", - "description_default": "locale.wc.default", - "prefix": "https://www.wechat.com/", - "method_type": "third_party", - "controller": "wechat.com" - } - }, - "xi": { - "rewriteKey": "xing", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.xi.name", - "description_default": "locale.xi.default", - "prefix": "https://www.xing.com/", - "method_type": "third_party", - "controller": "xing.com" - } - }, - "yy": { - "rewriteKey": "yy", - "assignKeys": [ - "v", - "d" - ], - "rewriteValue": { - "object_display_name": "locale.yy.name", - "description_default": "locale.yy.default", - "prefix": "https://www.yy.com/", - "method_type": "third_party", - "controller": "yy.com" - } - } - }`, - Output: `{ - "@n": 1, - "organisation": { - "object_display_name": "Organisation", - "name": "NUM Example Co", - "slogan": "Example Strapline", - "contacts": [ - { - "telephone": { - "object_display_name": "Telephone", - "description_default": "Call", - "description": "Customer Service", - "prefix": "tel:", - "method_type": "core", - "value": "+441270123456", - "hours": null - } - }, - { - "facebook": { - "object_display_name": "Facebook", - "description_default": "View Facebook profile", - "description": null, - "prefix": "https://www.facebook.com/", - "method_type": "third_party", - "controller": "facebook.com", - "value": "examplefacebook" - } - }, - { - "instagram": { - "object_display_name": "Instagram", - "description_default": "View Instagram profile", - "description": null, - "prefix": "https://www.instagram.com/", - "method_type": "third_party", - "controller": "instagram.com", - "value": "exampleinstagram" - } - }, - { - "twitter": { - "object_display_name": "Twitter", - "description_default": "View Twitter profile", - "description": null, - "prefix": "https://www.twitter.com/", - "method_type": "third_party", - "controller": "twitter.com", - "value": "exampletwitter", - "value_prefix": "@" - } - } - ] - } - }`, - }, - - // Source: https://www.unpacker.uk/playground?e=2 - { - Name: "playground-2", - Input: `{ - "?": ["https://www.widgetcompany.com", "team", "janesmith", "johnwilson", "dashnaanand"], - "o": [ - "Widget Company Ltd", - "Making the best widgets", - "%0", - [ - ["Jane Smith", "%ceo", "%0%/%1%/%2", "%2", "%2%widgets"], - ["John Wilson", "%cto", "%0%/%1%/%3", "%3", "jono"], - ["Dashna Anand", "%cmo", "%0%/%1%/%4", "%4", "%4"] - ] - ] - }`, - Subs: `{ - "ceo": "Chief Executive Officer", - "cto": "Chief Technology Officer", - "cmo": "Chief Marketing Officer" - }`, - Trans: `{ - "o": { - "assignKeys": ["n", "s", "w", "e"], - "replacePair": { - "name": "%n", - "strapline": "%s", - "website": "%w", - "employees": "%e" - } - }, - "em": { - "replacePair": { - "name": "%n", - "position": "%p", - "bio": "%b", - "linkedin": "https://www.linkedin.com/in/%l", - "twitter": "https://www.twitter.com/%t" - } - } - }`, - Output: `{ - "name": "Widget Company Ltd", - "strapline": "Making the best widgets", - "website": "https://www.widgetcompany.com", - "employees": [ - [ - "Jane Smith", - "Chief Executive Officer", - "https://www.widgetcompany.com/team/janesmith", - "janesmith", - "janesmithwidgets" - ], - [ - "John Wilson", - "Chief Technology Officer", - "https://www.widgetcompany.com/team/johnwilson", - "johnwilson", - "jono" - ], - [ - "Dashna Anand", - "Chief Marketing Officer", - "https://www.widgetcompany.com/team/dashnaanand", - "dashnaanand", - "dashnaanand" - ] - ] - }`, - }, - - // Source: https://www.unpacker.uk/playground?e=1 - { - Name: "playground-1", - Input: `{ - "?": ["abccompany", "+4412345678"], - "c": "ABC Company Ltd", - "t": [ - {"l": "%ac", "n": "%1%90"}, - {"l": "%cs", "n": "%1%89"} - ], - "tw": "/%0", - "i": "/%0%pics" - }`, - Subs: `{"cs": "Customer Service", "ac": "Accounts"}`, - Trans: `{ - "c": {"rewriteKey": "coname"}, - "t": {"rewriteKey": "telephone"}, - "l": {"rewriteKey": "label"}, - "n": {"rewriteKey": "number"}, - "tw": {"rewriteKey": "twitter"}, - "i": {"rewriteKey": "instagram"} - }`, - Output: `{ - "coname": "ABC Company Ltd", - "telephone": [ - {"label": "Accounts", "number": "+441234567890"}, - {"label": "Customer Service", "number": "+441234567889"} - ], - "twitter": "/abccompany", - "instagram": "/abccompanypics" - }`, - }, - - // Source: https://www.unpacker.uk/specification#variable-index - { - Name: "variable-index-1", - Input: `{"?": [1], "a": "%0%"}`, - Output: `{"a": 1}`, - }, - { - Name: "variable-index-2", - Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0%", "y": "%1%"}`, - Output: `{"x": ["a", "b"], "y": {"a": "alpha", "b": "bravo"}}`, - }, - { - Name: "variable-index-3", - Input: `{"?": [["a", "b"], {"a": "alpha", "b": "bravo"}], "x": "%0.0%", "y": "%1.a%"}`, - Output: `{"x": "a", "y": "alpha"}`, - }, - - // Source: https://www.unpacker.uk/specification#substitution-object - { - Name: "sub-1", - Input: `{"this": "%a%", "that": "%b%"}`, - Subs: `{"a": 1, "b": 2}`, - Output: `{"this": 1, "that": 2}`, - }, - { - Name: "sub-2", - Input: `{"this": "%a.x%", "that": "%b.0%"}`, - Subs: `{"a": {"x": "xray", "y": "yankee"}, "b": [1, 2]}`, - Output: `{"this": "xray", "that": 1}`, - }, - - // Source: https://www.unpacker.uk/specification#transformation-object - { - Name: "trans-1", - Input: `{ - "t": 1, - "u": {"t": 2}, - "v": {"w": {"t": 3}} - }`, - Trans: `{"t": {"rewriteKey": "testing"}}`, - Output: `{ - "testing": 1, - "u": {"testing": 2}, - "v": {"w": {"testing": 3}} - }`, - }, - { - Name: "trans-2", - Input: `{ - "t": 1, - "u": {"t": 2}, - "v": {"w": {"t": 3}} - }`, - Trans: `{ - "t": {"rewriteKey": "testing"}, - "u": {"t": {"rewriteKey": "testing2"}}, - "v": {"w": {"t": {"rewriteKey": "testing3"}}} - }`, - Output: `{ - "testing": 1, - "u": {"testing2": 2}, - "v": {"w": {"testing3": 3}} - }`, - }, - { - Name: "trans-3", - Input: `{"t": 1}`, - Trans: `{"t": {"rewriteKey": "testing"}}`, - Output: `{"testing": 1}`, - }, - { - Name: "trans-4", - Input: `{"t": "me"}`, - Trans: `{ - "t": { - "rewriteKey": "twitter", - "rewriteValue": "twitter.com/%self" - } - }`, - Output: `{"twitter": "twitter.com/me"}`, - }, - { - Name: "trans-5", - Input: `{"t": "me"}`, - Trans: `{ - "t": { - "rewriteKey": "twitter", - "rewriteValue": { - "cta": "Follow on Twitter", - "url": "twitter.com/%self" - } - } - }`, - Output: `{ - "twitter": { - "cta": "Follow on Twitter", - "url": "twitter.com/me" - } - }`, - }, - { - Name: "trans-6", - Input: `{"t": "me"}`, - Trans: `{ - "t": { - "replacePair": { - "cta": "Follow on Twitter", - "url": "twitter.com/%self" - } - } - }`, - Output: `{"cta": "Follow on Twitter", "url": "twitter.com/me"}`, - }, - { - Name: "trans-7", - Input: `{"phonetic": ["alpha", "bravo", "charlie"]}`, - Trans: `{"phonetic": {"assignKeys": ["a", "b", "c"]}}`, - Output: `{ - "phonetic": { - "a": "alpha", - "b": "bravo", - "c": "charlie" - } - }`, - }, - { - Name: "trans-8", - Input: `{ - "employees": [ - ["Jane Smith", "CEO"], - ["Dashna Anand", "CMO"] - ] - }`, - Trans: `{ - "employees": { - "arrayItems": "employee" - }, - "employee": { - "assignKeys": [ - "name", - "position" - ], - "replacePair": { - "name": "%name", - "position": "%position" - } - } - }`, - Output: `{"employees": [ - {"name": "Jane Smith", "position": "CEO"}, - {"name": "Dashna Anand", "position": "CMO"} - ]}`, - }, - - // Source: https://www.unpacker.uk/specification#referencing - { - Name: "ref-1", - Input: `{ - "?": ["test"], - "a": "%0%", - "b": "this is a %0", - "c": "this is another %0 of referencing", - "d": "once again %0%ing it" - }`, - Output: `{ - "a": "test", - "b": "this is a test", - "c": "this is another test of referencing", - "d": "once again testing it" - }`, - }, - { - Name: "ref-2", - Input: `{ - "?": [ - ["a", "b"], - {"a": "alpha", "b": "bravo"} - ], - "x": "%0.0", - "y": "%1.a", - "z": "%letters.0" - }`, - Subs: `{"letters": ["a", "b"]}`, - Output: `{"x": "a", "y": "alpha", "z": "a"}`, - }, - { - Name: "ref-3", - Input: `{"a": {"b": {"c": 1}}}`, - Trans: `{"a": {"rewriteValue": "%b.c"}}`, - Output: `{"a": 1}`, - }, - { - Name: "ref-4", - Input: `{ - "a": ["alpha", "bravo", "charlie"] - }`, - Trans: `{ - "a": { - "assignKeys": ["a", "b", "c"], - "rewriteValue": "%c" - } - }`, - Output: `{"a": "charlie"}`, - }, - { - Name: "ref-5", - Input: `{"a": 1, "b": 2, "c": 3}`, - Trans: `{ - "a": {"rewriteValue": "%/compact.c"}, - "b": {"rewriteValue": "%/subs.x"} - }`, - Subs: `{"x": 4}`, - Output: `{"a": 3, "b": 4, "c": 3}`, - }, -} - type UnpackerTest struct { - Name string - Input string - Subs string - Trans string - Output string + ID string + Input json.RawMessage + Subs json.RawMessage + Trans json.RawMessage + Output json.RawMessage Skip bool } @@ -1347,7 +21,7 @@ func (test *UnpackerTest) Run(t *testing.T) { t.Skip("manually disabled test") } u := &Unpacker{} - if test.Trans != `` { + if len(test.Trans) != 0 { trans, err := ParseTransforms([]byte(test.Trans)) if err != nil { t.Fatalf("Unable to parse transforms: %v", err) @@ -1355,7 +29,7 @@ func (test *UnpackerTest) Run(t *testing.T) { u.Transforms = trans // fmt.Printf("Got Transforms: %v\n", trans) } - if test.Subs != `` { + if len(test.Subs) != 0 { subs := make(Substitution) if err := json.Unmarshal([]byte(test.Subs), &subs); err != nil { t.Fatalf("Unable to parse substition: %v", err) @@ -1385,8 +59,28 @@ func (test UnpackerTest) out() []byte { return bits } +var skip = map[string]bool{ + "Unpacker-07": true, // transform with variadic-index reference + "Unpacker-09": true, // replacePair: null + "Unpacker-10": true, // rewriteValue: null + "Unpacker-13": true, + "Unpacker-14": true, // transforms with reserved-keys alongside user-keys + "Nested-01": true, // transforms with reserved-keys alongside user-keys + "Nested-03": true, // transforms with reserved-keys alongside user-keys + "Unpacker-": true, +} + func TestUnpacker(t *testing.T) { - for _, test := range unpackerTests { - t.Run(test.Name, test.Run) + jsonTests, err := ioutil.ReadFile("tests.json") + if err != nil { + t.Skip("Cannot read base_tests.json (git submodule init): " + err.Error()) + } + var tests []UnpackerTest + if err = json.Unmarshal(jsonTests, &tests); err != nil { + t.Fatal("Unable to deserialize tests: " + err.Error()) + } + for _, test := range tests { + test.Skip = skip[test.ID] + t.Run(test.ID, test.Run) } } From c12e8938e5d23875f4315364e44531786de527f9 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Mon, 24 May 2021 00:54:34 +0000 Subject: [PATCH 15/27] Use custom Transform JSON Marshalers for null case Go does not have a way to detect if a key is present but assigned null. To work around this, Null booleans were added to `Transforms`. Custom serialization logic that uses a generic `map[string]interface{}` then sets these feilds appropriately using `v, ok := map[key]`. --- unpacker/unpacker.go | 137 ++++++++++++++++++++++++-------------- unpacker/unpacker_test.go | 8 +-- 2 files changed, 89 insertions(+), 56 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 1ab5e44..a76ffc0 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -20,59 +20,100 @@ type Unpacker struct { // A Transform holds information on how to modify various JSON objects. // // docs: https://www.unpacker.uk/specification#transformation-object +// +// Note: field tags are included on this object for convience as the real work is performed in Un/MarshalJSON. type Transform struct { - Assign []string `json:"assignKeys,omitempty"` - Items string `json:"arrayItems,omitempty"` - Key string `json:"rewriteKey,omitempty"` - Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) - Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object - Nesting map[string]Transform `json:"-"` + Assign []string `json:"assignKeys,omitempty"` + Items string `json:"arrayItems,omitempty"` + Key string `json:"rewriteKey,omitempty"` + Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) + ReturnNull bool `json:"-"` + Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object + RewriteNull bool `json:"-"` + Nesting map[string]Transform `json:"-"` } -func (t Transform) String() string { - x, err := json.Marshal(trans(t)) - if err != nil { - return err.Error() +func (t *Transform) UnmarshalJSON(bits []byte) error { + // TODO: emit errors if the types are wrong + trans := make(map[string]interface{}) // todo: map string to json.RawMessage? + if err := json.Unmarshal(bits, &trans); err != nil { + return err } - return string(x) -} - -// internal for now but could be exposed, makes it so we can parse nested transforms without infinite recursive loops -type trans Transform - -func (t *trans) UnmarshalJSON(bits []byte) error { - x := Transform{} - dec := json.NewDecoder(bytes.NewReader(bits)) - dec.DisallowUnknownFields() - err := dec.Decode(&x) - - // attempt to recurse through nested instructions - if err != nil { - x.Nesting, err = ParseTransforms(bits) + if assign, ok := trans["assignKeys"].([]interface{}); ok { + t.Assign = make([]string, 0, len(assign)) + for _, a := range assign { + t.Assign = append(t.Assign, a.(string)) + } + delete(trans, "assignKeys") + } + if items, ok := trans["arrayItems"].(string); ok { + t.Items = items + delete(trans, "arrayItems") + } + if key, ok := trans["rewriteKey"].(string); ok { + t.Key = key + delete(trans, "rewriteKey") + } + if replace, ok := trans["replacePair"]; ok { + if replace == nil { + t.ReturnNull = true + } else if rep, ok := replace.(map[string]interface{}); ok { + t.Return = rep + } + delete(trans, "replacePair") + } + if rewrite, ok := trans["rewriteValue"]; ok { + if rewrite == nil { + t.RewriteNull = true + } else { + t.Rewrite = rewrite + } + delete(trans, "rewriteValue") + } + if len(trans) > 0 { + t.Nesting = make(map[string]Transform) + bits, err := json.Marshal(trans) if err != nil { return err } + return json.Unmarshal(bits, &t.Nesting) } - *t = trans(x) return nil } -// Requies some weirdness to encode the Nested definitions as well -func (t trans) MarshalJSON() ([]byte, error) { - bits, err := json.Marshal(Transform(t)) - if err != nil || len(t.Nesting) == 0 { - return bits, err +func (t Transform) MarshalJSON() ([]byte, error) { + trans := make(map[string]interface{}) + if len(t.Assign) > 0 { + trans["assignKeys"] = t.Assign + } + if t.Items != "" { + trans["arrayItems"] = t.Items + } + if t.Key != "" { + trans["rewriteKey"] = t.Key + } + if t.ReturnNull { + trans["replacePair"] = nil + } else if t.Return != nil { + trans["replacePair"] = t.Return + } + if t.RewriteNull { + trans["rewriteValue"] = nil + } else if t.Rewrite != nil { + trans["rewriteValue"] = t.Rewrite } - nest := make(map[string]trans, len(t.Nesting)) for k, v := range t.Nesting { - nest[k] = trans(v) + trans[k] = v } - nesting, err := json.Marshal(nest) + return json.Marshal(trans) +} + +func (t Transform) String() string { + x, err := json.Marshal(t) if err != nil { - return nil, err + return err.Error() } - bits = append(bits[:len(bits)-1], nesting[1:]...) - return bits, nil + return string(x) } // A Substitution object provides a method of removing repeditive data by having shortened keys. @@ -84,17 +125,8 @@ type Substitution map[string]interface{} // Transforms can be embeded, so the bulk of this logic is to properly parse embedded transforms. // Example: {"k": {"e": {"y": {/*transform object*/}}}} func ParseTransforms(transforms []byte) (map[string]Transform, error) { - v := map[string]trans{} - err := json.Unmarshal(transforms, &v) - if err != nil { - fmt.Printf("failed parsing: %s\n", string(transforms)) - return nil, err - } - u := make(map[string]Transform, len(v)) - for k, x := range v { - u[k] = Transform(x) - } - return u, err + v := map[string]Transform{} + return v, json.Unmarshal(transforms, &v) } func (u *Unpacker) AddSubs(subs Substitution) { @@ -315,7 +347,9 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // fmt.Printf("Got context: %v\n", ctx) // 2: replacePair and exit - if trans.Return != nil { + if trans.ReturnNull { + return // if replacePair is null, the key should not be assigned + } else if trans.Return != nil { for k, v := range trans.Return { // TODO: smarter string resolves if s, ok := v.(string); ok { @@ -327,7 +361,10 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu } // 3: rewriteValue - if trans.Rewrite != nil { + if trans.RewriteNull { + dest[key] = nil // if rewriteValue is null, assign value to null + return + } else if trans.Rewrite != nil { if m, ok := trans.Rewrite.(map[string]interface{}); ok { n := map[string]interface{}{} // ensure we don't duplicate the object for k, v := range m { diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 34a3b1e..115b7b9 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -61,12 +61,8 @@ func (test UnpackerTest) out() []byte { var skip = map[string]bool{ "Unpacker-07": true, // transform with variadic-index reference - "Unpacker-09": true, // replacePair: null - "Unpacker-10": true, // rewriteValue: null - "Unpacker-13": true, - "Unpacker-14": true, // transforms with reserved-keys alongside user-keys - "Nested-01": true, // transforms with reserved-keys alongside user-keys - "Nested-03": true, // transforms with reserved-keys alongside user-keys + "Unpacker-13": true, // arrayItems + nested + "Unpacker-14": true, // arrayItems + nested "Unpacker-": true, } From 9d7213a9f43ea60d61dbc1daef10693eb03671c7 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Mon, 24 May 2021 01:31:49 +0000 Subject: [PATCH 16/27] Transforms MUST NOT resolve with variadic-index --- unpacker/unpacker.go | 51 ++++++++++++++++++--------------------- unpacker/unpacker_test.go | 19 ++++----------- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index a76ffc0..325ae56 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -149,7 +149,10 @@ func (u *Unpacker) AddTransform(key string, trans Transform) { // // docs: https://www.unpacker.uk/specification func (u Unpacker) Unpack(source []byte) ([]byte, error) { - state := &unpackState{mem: make(map[string]interface{})} + state := &unpackState{ + fullMem: make(map[string]interface{}), + noVariMem: make(map[string]interface{}), + } state.trans = append(state.trans, u.Transforms) // pass one, extract the variable-index and memoize everything for `/compact.` namespaced vars @@ -160,11 +163,14 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { // memoize string replacements if obj, ok := compact.(map[string]interface{}); ok { - augment("", obj["?"], state.mem) - } - augment("", u.Subs, state.mem) - augment("/subs.", u.Subs, state.mem) - augment("/compact.", compact, state.mem) + augment("", obj["?"], state.fullMem) + } + augment("", u.Subs, state.fullMem) + augment("", u.Subs, state.noVariMem) + augment("/subs.", u.Subs, state.fullMem) + augment("/subs.", u.Subs, state.noVariMem) + augment("/compact.", compact, state.fullMem) + augment("/compact.", compact, state.noVariMem) // fmt.Printf("Values: %#v\n", state.mem) // pass two, process the data @@ -175,9 +181,10 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { } type unpackState struct { - dec *json.Decoder - mem map[string]interface{} - trans []map[string]Transform + dec *json.Decoder + fullMem map[string]interface{} + noVariMem map[string]interface{} + trans []map[string]Transform } func (state unpackState) value() interface{} { @@ -198,7 +205,7 @@ func (state unpackState) value() interface{} { } case string: // fmt.Printf("Got a string value: %q\n", token) - return state.string(v, nil) + return state.string(v, state.fullMem) case float64: // fmt.Printf("Got an int value: %f\n", token) return v @@ -245,23 +252,8 @@ func (state unpackState) array() []interface{} { } func (state unpackState) string(in string, subz map[string]interface{}) interface{} { - for k, v := range state.mem { - key := "%" + k - if in == key || in == key+"%" { - return v - } - if s, ok := v.(string); ok { - if strings.HasSuffix(in, key) { - in = strings.TrimSuffix(in, key) + s - } - if strings.Contains(in, key+" ") { - in = strings.Replace(in, key+" ", s+" ", -1) - } - if strings.Contains(in, key+"%") { - in = strings.Replace(in, key+"%", s, -1) - } - } - } + // NOTE: state is not used, but is left as a potential memory optimization + // (array of contexts or logic to switch between allowing viariadic or not) for k, v := range subz { key := "%" + k if in == key || in == key+"%" { @@ -319,7 +311,10 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu } // 0.5: Start building context (assign keys can modify some default values here) - ctx := map[string]interface{}{} + ctx := make(map[string]interface{}, len(state.fullMem)) + for k, v := range state.noVariMem { + ctx[k] = v + } // 1. Assign(assignKeys) converts arrays to objects if list, is_list := value.([]interface{}); len(trans.Assign) != 0 && is_list { diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 115b7b9..610c2ed 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -10,8 +10,8 @@ import ( type UnpackerTest struct { ID string Input json.RawMessage - Subs json.RawMessage - Trans json.RawMessage + Subs Substitution + Trans map[string]Transform Output json.RawMessage Skip bool } @@ -22,19 +22,11 @@ func (test *UnpackerTest) Run(t *testing.T) { } u := &Unpacker{} if len(test.Trans) != 0 { - trans, err := ParseTransforms([]byte(test.Trans)) - if err != nil { - t.Fatalf("Unable to parse transforms: %v", err) - } - u.Transforms = trans - // fmt.Printf("Got Transforms: %v\n", trans) + u.Transforms = test.Trans + // fmt.Printf("Got Transforms: %v\n", u.Transforms) } if len(test.Subs) != 0 { - subs := make(Substitution) - if err := json.Unmarshal([]byte(test.Subs), &subs); err != nil { - t.Fatalf("Unable to parse substition: %v", err) - } - u.AddSubs(subs) + u.AddSubs(test.Subs) } output, err := u.Unpack([]byte(test.Input)) if err != nil { @@ -60,7 +52,6 @@ func (test UnpackerTest) out() []byte { } var skip = map[string]bool{ - "Unpacker-07": true, // transform with variadic-index reference "Unpacker-13": true, // arrayItems + nested "Unpacker-14": true, // arrayItems + nested "Unpacker-": true, From 29355d0150153cc161d2a00e8aaa6b8609773569 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Mon, 24 May 2021 11:51:05 +0000 Subject: [PATCH 17/27] Array Trans Items rewritePair w/string --- unpacker/unpacker.go | 51 ++++++++++++++++++++++++++++++++++++--- unpacker/unpacker_test.go | 1 - 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 325ae56..3ec4d4d 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -28,6 +28,7 @@ type Transform struct { Key string `json:"rewriteKey,omitempty"` Return map[string]interface{} `json:"replacePair,omitempty"` // can be "null" to nuke a pair (how?) ReturnNull bool `json:"-"` + ReturnStr string `json:"-"` // replacePair: "%self" Rewrite interface{} `json:"rewriteValue,omitempty"` // freeform object RewriteNull bool `json:"-"` Nesting map[string]Transform `json:"-"` @@ -57,6 +58,8 @@ func (t *Transform) UnmarshalJSON(bits []byte) error { if replace, ok := trans["replacePair"]; ok { if replace == nil { t.ReturnNull = true + } else if str, ok := replace.(string); ok { + t.ReturnStr = str } else if rep, ok := replace.(map[string]interface{}); ok { t.Return = rep } @@ -94,6 +97,8 @@ func (t Transform) MarshalJSON() ([]byte, error) { } if t.ReturnNull { trans["replacePair"] = nil + } else if t.ReturnStr != "" { + trans["replacePair"] = t.ReturnStr } else if t.Return != nil { trans["replacePair"] = t.Return } @@ -222,16 +227,31 @@ func (state unpackState) object() map[string]interface{} { k := state.value().(string) // Push transform state (for use when processing value) + var has_arr_trans bool trans, has_trans := state.getTransform(k) if has_trans { state.trans = append(state.trans, trans.Nesting) } + if has_trans && trans.Items != "" { + var arr_trans Transform + arr_trans, has_arr_trans = state.getTransform(trans.Items) + if has_arr_trans { + state.trans = append(state.trans, arr_trans.Nesting) + } + } + // Parse the value! v := state.value() - if k == "?" { - continue + + // Pop arr_trans + if has_arr_trans { + state.trans = state.trans[:len(state.trans)-1] + } + + // if we aren't the variadic index, transform data (time saver?) + if k != "?" { + state.transform(o, k, v) } - state.transform(o, k, v) // Pop transform state if has_trans { @@ -301,11 +321,27 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu if !ok { panic(fmt.Sprintf("arrayItems set need an array; %T", value)) } + + // Push transform state (for use when processing value) + sub_trans, has_trans := state.getTransform(trans.Items) + if has_trans { + state.trans = append(state.trans, map[string]Transform{ + trans.Items: sub_trans, + }) + } + + // todo: push context of trans.Items onto stack@! for i, v := range list { n := map[string]interface{}{} // TODO: preallocate size state.transform(n, trans.Items, v) list[i] = n } + + // Pop transform state + if has_trans { + state.trans = state.trans[:len(state.trans)-1] + } + dest[key] = list return // not much else we can do here } @@ -344,6 +380,15 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // 2: replacePair and exit if trans.ReturnNull { return // if replacePair is null, the key should not be assigned + } else if trans.ReturnStr != "" { + obj := state.string(trans.ReturnStr, ctx) + if m, ok := obj.(map[string]interface{}); ok { + for k, v := range m { + dest[k] = v + } + return + } + panic(fmt.Sprintf("Got unexpected ReturnString type: %T", obj)) } else if trans.Return != nil { for k, v := range trans.Return { // TODO: smarter string resolves diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 610c2ed..708f6c1 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -53,7 +53,6 @@ func (test UnpackerTest) out() []byte { var skip = map[string]bool{ "Unpacker-13": true, // arrayItems + nested - "Unpacker-14": true, // arrayItems + nested "Unpacker-": true, } From ac6b5aec05981730127b0361ccf5b35395fe0cd8 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Tue, 25 May 2021 05:14:17 +0000 Subject: [PATCH 18/27] allow further mods on array transforms --- unpacker/unpacker.go | 3 +-- unpacker/unpacker_test.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 3ec4d4d..44dd2dd 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -342,8 +342,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu state.trans = state.trans[:len(state.trans)-1] } - dest[key] = list - return // not much else we can do here + value = list } // 0.5: Start building context (assign keys can modify some default values here) diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 708f6c1..2ff05e4 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -52,8 +52,7 @@ func (test UnpackerTest) out() []byte { } var skip = map[string]bool{ - "Unpacker-13": true, // arrayItems + nested - "Unpacker-": true, + "Unpacker-": true, } func TestUnpacker(t *testing.T) { From 79a0d7e8cdf42ea629126d040bf0163a8143f02c Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Tue, 25 May 2021 15:15:50 +0000 Subject: [PATCH 19/27] pulling over some more testcases --- unpacker/schema-map.json | 760 ++++++++++++++++++++++++++++++++++++++ unpacker/tests.json | 161 +++++++- unpacker/unpacker_test.go | 38 +- 3 files changed, 948 insertions(+), 11 deletions(-) create mode 100644 unpacker/schema-map.json diff --git a/unpacker/schema-map.json b/unpacker/schema-map.json new file mode 100644 index 0000000..90b451f --- /dev/null +++ b/unpacker/schema-map.json @@ -0,0 +1,760 @@ +{ + "@n": { + "rewriteValue": "%/compact.@n" + }, + "o": { + "rewriteKey": "organisation", + "assignKeys": [ + "n", + "s", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.o.name", + "name": "%n", + "slogan": "%s", + "contacts": "%c" + } + }, + "p": { + "rewriteKey": "person", + "assignKeys": [ + "n", + "b", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.p.name", + "name": "%n", + "bio": "%b", + "contacts": "%c" + } + }, + "e": { + "rewriteKey": "employee", + "assignKeys": [ + "n", + "r", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.e.name", + "name": "%n", + "role": "%r", + "contacts": "%c" + } + }, + "lc": { + "rewriteKey": "location", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.lc.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "gp": { + "rewriteKey": "group", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.gp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "dp": { + "rewriteKey": "department", + "assignKeys": [ + "n", + "d", + "c" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.dp.name", + "name": "%n", + "description": "%d", + "contacts": "%c" + } + }, + "a": { + "rewriteKey": "address", + "assignKeys": [ + "al", + "pz", + "co", + "d" + ], + "rewriteValue": { + "description": "%d", + "description_default": "%/subs.locale.a.default", + "lines": "%al", + "postcode": "%pz", + "country": "%co", + "method_type": "core", + "object_display_name": "%/subs.locale.a.name", + "prefix": "" + } + }, + "l": { + "rewriteKey": "link", + "assignKeys": [ + "@L", + "d", + "n" + ], + "rewriteValue": { + "@L": "%@L", + "description": "%d", + "name": "%n", + "object_display_name": "Link" + } + }, + "fb": { + "rewriteKey": "facebook", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fb.name", + "description_default": "%/subs.locale.fb.default", + "description": "%d", + "prefix": "https://www.facebook.com/", + "method_type": "third_party", + "controller": "facebook.com", + "value": "%v" + } + }, + "g": { + "rewriteKey": "gps", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.g.name", + "description_default": "%/subs.locale.g.default", + "description": "%d", + "prefix": "", + "method_type": "core", + "value": "%v" + } + }, + "in": { + "rewriteKey": "instagram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.in.name", + "description_default": "%/subs.locale.in.default", + "description": "%d", + "prefix": "https://www.instagram.com/", + "method_type": "third_party", + "controller": "instagram.com", + "value": "%v" + } + }, + "li": { + "rewriteKey": "linkedin", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.li.name", + "description_default": "%/subs.locale.li.default", + "description": "%d", + "prefix": "https://www.linkedin.com/", + "method_type": "third_party", + "controller": "linkedin.com", + "value": "%v" + } + }, + "yt": { + "rewriteKey": "youtube", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.yt.name", + "description_default": "%/subs.locale.yt.default", + "description": "%d", + "prefix": "https://www.youtube.com/", + "method_type": "third_party", + "controller": "youtube.com", + "value": "%v" + } + }, + "pi": { + "rewriteKey": "pinterest", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.pi.name", + "description_default": "%/subs.locale.pi.default", + "description": "%d", + "prefix": "https://www.pinterest.com/", + "method_type": "third_party", + "controller": "pinterest.com", + "value": "%v" + } + }, + "tw": { + "rewriteKey": "twitter", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.tw.name", + "description_default": "%/subs.locale.tw.default", + "description": "%d", + "prefix": "https://www.twitter.com/", + "method_type": "third_party", + "controller": "twitter.com", + "value": "%v", + "value_prefix": "@" + } + }, + "t": { + "rewriteKey": "telephone", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.t.name", + "description_default": "%/subs.locale.t.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "sm": { + "rewriteKey": "sms", + "assignKeys": [ + "v", + "d", + "h" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.sm.name", + "description_default": "%/subs.locale.sm.default", + "description": "%d", + "prefix": "sms:", + "method_type": "core", + "value": "%v", + "hours": "%h" + } + }, + "em": { + "rewriteKey": "email", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.em.name", + "description_default": "%/subs.locale.em.default", + "description": "%d", + "prefix": "mailto:", + "method_type": "core", + "value": "%v" + } + }, + "fx": { + "rewriteKey": "fax", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.fx.name", + "description_default": "%/subs.locale.fx.default", + "description": "%d", + "prefix": "tel:", + "method_type": "core", + "value": "%v" + } + }, + "u": { + "rewriteKey": "url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.u.name", + "description_default": "%/subs.locale.u.default", + "description": "%d", + "prefix": "https://", + "method_type": "core", + "value": "%v" + } + }, + "uu": { + "rewriteKey": "unsecure_url", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "%/subs.locale.uu.name", + "description_default": "%/subs.locale.uu.default", + "description": "%d", + "prefix": "http://", + "method_type": "core", + "value": "%v" + } + }, + "av": { + "rewriteKey": "available" + }, + "tz": { + "rewriteKey": "time_zone_location" + }, + "i": { + "rewriteKey": "introduction" + }, + "ac": { + "rewriteKey": "access" + }, + "aa": { + "rewriteKey": "android-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.aa.name", + "description_default": "locale.aa.default", + "prefix": "https://play.google.com/store/apps/details?id=", + "method_type": "third_party", + "controller": "play.google.com" + } + }, + "as": { + "rewriteKey": "ios-app", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.as.name", + "description_default": "locale.as.default", + "prefix": "https://itunes.apple.com/app/", + "method_type": "third_party", + "controller": "apps.apple.com" + } + }, + "bt": { + "rewriteKey": "baidu_tieba", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.bt.name", + "description_default": "locale.bt.default", + "prefix": "https://tieba.baidu.com/", + "method_type": "third_party", + "controller": "tieba.baidu.com" + } + }, + "fs": { + "rewriteKey": "foursquare", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.fs.name", + "description_default": "locale.fs.default", + "prefix": "https://www.foursquare.com/", + "method_type": "third_party", + "controller": "foursquare.com" + } + }, + "ft": { + "rewriteKey": "facetime", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ft.name", + "description_default": "locale.ft.default", + "prefix": "facetime://", + "method_type": "third_party", + "controller": "facetime@apple.com" + } + }, + "gh": { + "rewriteKey": "github", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.gh.name", + "description_default": "locale.gh.default", + "prefix": "https://www.github.com/", + "method_type": "third_party", + "controller": "github.com" + } + }, + "im": { + "rewriteKey": "imessage", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.im.name", + "description_default": "locale.im.default", + "prefix": "imessage://", + "method_type": "third_party", + "controller": "imessage@apple.com" + } + }, + "kk": { + "rewriteKey": "kik", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.kk.name", + "description_default": "locale.kk.default", + "prefix": "https://www.kik.com/u/", + "method_type": "third_party", + "controller": "kik.com" + } + }, + "ln": { + "rewriteKey": "line", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.ln.name", + "description_default": "locale.ln.default", + "prefix": "line://", + "method_type": "third_party", + "controller": "line.me" + } + }, + "md": { + "rewriteKey": "medium", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.md.name", + "description_default": "locale.md.default", + "prefix": "https://www.medium.com/", + "method_type": "third_party", + "controller": "medium.com" + } + }, + "pr": { + "rewriteKey": "periscope", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.pr.name", + "description_default": "locale.pr.default", + "prefix": "https://www.periscope.tv/", + "method_type": "third_party", + "controller": "periscope.tv" + } + }, + "qq": { + "rewriteKey": "qq", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qq.name", + "description_default": "locale.qq.default", + "prefix": "https://www.qq.com/", + "method_type": "third_party", + "controller": "qq.com" + } + }, + "qz": { + "rewriteKey": "qzone", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.qz.name", + "description_default": "locale.qz.default", + "prefix": "https://www.qzone.com/", + "method_type": "third_party", + "controller": "qzone.com" + } + }, + "rd": { + "rewriteKey": "reddit", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rd.name", + "description_default": "locale.rd.default", + "prefix": "https://www.reddit.com/r/", + "method_type": "third_party", + "controller": "reddit.com" + } + }, + "rn": { + "rewriteKey": "renren", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.rn.name", + "description_default": "locale.rn.default", + "prefix": "https://www.renren.com/", + "method_type": "third_party", + "controller": "renren.com" + } + }, + "sc": { + "rewriteKey": "soundcloud", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sc.name", + "description_default": "locale.sc.default", + "prefix": "https://www.soundcloud.com/", + "method_type": "third_party", + "controller": "soundcloud.com" + } + }, + "sk": { + "rewriteKey": "skype", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sk.name", + "description_default": "locale.sk.default", + "prefix": "skype:", + "method_type": "third_party", + "controller": "skype.com" + } + }, + "sr": { + "rewriteKey": "swarm", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sr.name", + "description_default": "locale.sr.default", + "prefix": "https://www.swarmapp.com/", + "method_type": "third_party", + "controller": "swarmapp.com" + } + }, + "sn": { + "rewriteKey": "snapchat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sn.name", + "description_default": "locale.sn.default", + "prefix": "snapchat://add/", + "method_type": "third_party", + "controller": "snapchat.com" + } + }, + "sw": { + "rewriteKey": "sina-weibo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.sw.name", + "description_default": "locale.sw.default", + "prefix": "https://www.weibo.com/", + "method_type": "third_party", + "controller": "weibo.com" + } + }, + "tb": { + "rewriteKey": "tumblr", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tb.name", + "description_default": "locale.tb.default", + "prefix": "https://.tumblr.com/", + "method_type": "third_party", + "controller": "tumblr.com" + } + }, + "tl": { + "rewriteKey": "telegram", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.tl.name", + "description_default": "locale.tl.default", + "prefix": "https://www.telegram.me/", + "method_type": "third_party", + "controller": "telegram.com" + } + }, + "to": { + "rewriteKey": "twoo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.to.name", + "description_default": "locale.to.default", + "prefix": "https://www.twoo.com/", + "method_type": "third_party", + "controller": "twoo.com" + } + }, + "vb": { + "rewriteKey": "viber", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vb.name", + "description_default": "locale.vb.default", + "prefix": "https://www.viber.com/", + "method_type": "third_party", + "controller": "viber.com" + } + }, + "vk": { + "rewriteKey": "vkontakte", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vk.name", + "description_default": "locale.vk.default", + "prefix": "https://www.vk.com/", + "method_type": "third_party", + "controller": "vk.com" + } + }, + "vm": { + "rewriteKey": "vimeo", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.vm.name", + "description_default": "locale.vm.default", + "prefix": "https://www.vimeo.com/", + "method_type": "third_party", + "controller": "vimeo.com" + } + }, + "wa": { + "rewriteKey": "whatsapp", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wa.name", + "description_default": "locale.wa.default", + "prefix": "whatsapp://", + "method_type": "third_party", + "controller": "whatsapp.com" + } + }, + "wc": { + "rewriteKey": "wechat", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.wc.name", + "description_default": "locale.wc.default", + "prefix": "https://www.wechat.com/", + "method_type": "third_party", + "controller": "wechat.com" + } + }, + "xi": { + "rewriteKey": "xing", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.xi.name", + "description_default": "locale.xi.default", + "prefix": "https://www.xing.com/", + "method_type": "third_party", + "controller": "xing.com" + } + }, + "yy": { + "rewriteKey": "yy", + "assignKeys": [ + "v", + "d" + ], + "rewriteValue": { + "object_display_name": "locale.yy.name", + "description_default": "locale.yy.default", + "prefix": "https://www.yy.com/", + "method_type": "third_party", + "controller": "yy.com" + } + } +} \ No newline at end of file diff --git a/unpacker/tests.json b/unpacker/tests.json index 0956515..bb47d6f 100644 --- a/unpacker/tests.json +++ b/unpacker/tests.json @@ -1417,9 +1417,162 @@ }] } }, { - "id": "Unpacker-", - "desc": "", - "input": null, - "output": null + "id": "all-step-1", + "input": {"n":"Widget Company Ltd","s":"Making the best widgets","w":"https://www.widgetcompany.com","e":[{"n":"Jane Smith","p":"Chief Executive Officer","b":"https://www.widgetcompany.com/team/janesmith","l":"https://www.linkedin.com/janesmith","t":"https://www.twitter.com/janesmithwidgets"},{"n":"John Wilson","p":"Chief Technology Officer","b":"https://www.widgetcompany.com/team/johnwilson","l":"https://www.linkedin.com/in/johnwilson","t":"https://www.twitter.com/jono"},{"n":"Dashna Anand","p":"Chief Marketing Officer","b":"https://www.widgetcompany.com/team/dashnaanand","l":"https://www.linkedin.com/in/dashnaanand","t":"https://www.twitter.com/dashnaanand"}]}, + "trans": {"n":{"rewriteKey":"name"},"s":{"rewriteKey":"strapline"},"w":{"rewriteKey":"website"},"e":{"rewriteKey":"employees"},"p":{"rewriteKey":"position"},"b":{"rewriteKey":"bio"},"l":{"rewriteKey":"linkedin"},"t":{"rewriteKey":"twitter"}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-2", + "input": {"n":"Widget Company Ltd","s":"Making the best widgets","w":"https://www.widgetcompany.com","e":[{"n":"Jane Smith","p":"Chief Executive Officer","b":"https://www.widgetcompany.com/team/janesmith","l":"janesmith","t":"janesmithwidgets"},{"n":"John Wilson","p":"Chief Technology Officer","b":"https://www.widgetcompany.com/team/johnwilson","l":"johnwilson","t":"jono"},{"n":"Dashna Anand","p":"Chief Marketing Officer","b":"https://www.widgetcompany.com/team/dashnaanand","l":"dashnaanand","t":"dashnaanand"}]}, + "trans": {"n":{"rewriteKey":"name"},"s":{"rewriteKey":"strapline"},"w":{"rewriteKey":"website"},"e":{"rewriteKey":"employees"},"p":{"rewriteKey":"position"},"b":{"rewriteKey":"bio"},"l":{"rewriteKey":"linkedin","rewriteValue":"https://www.linkedin.com/in/%self"},"t":{"rewriteKey":"twitter","rewriteValue":"https://www.twitter.com/%self"}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-3a-1", + "input": {"o":["Widget Company Ltd","Making the best widgets","https://www.widgetcompany.com",[{"n":"Jane Smith","p":"Chief Executive Officer","b":"https://www.widgetcompany.com/team/janesmith","l":"janesmith","t":"janesmithwidgets"},{"n":"John Wilson","p":"Chief Technology Officer","b":"https://www.widgetcompany.com/team/johnwilson","l":"johnwilson","t":"jono"},{"n":"Dashna Anand","p":"Chief Marketing Officer","b":"https://www.widgetcompany.com/team/dashnaanand","l":"dashnaanand","t":"dashnaanand"}]]}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"rewriteValue":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"n":{"rewriteKey":"name"},"p":{"rewriteKey":"position"},"b":{"rewriteKey":"bio"},"l":{"rewriteKey":"linkedin","rewriteValue":"https://www.linkedin.com/in/%self"},"t":{"rewriteKey":"twitter","rewriteValue":"https://www.twitter.com/%self"}}, + "output": {"o":{"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]}} + }, { + "id": "all-step-3a-2", + "input": {"o":["Widget Company Ltd","Making the best widgets","https://www.widgetcompany.com",[{"n":"Jane Smith","p":"Chief Executive Officer","b":"https://www.widgetcompany.com/team/janesmith","l":"janesmith","t":"janesmithwidgets"},{"n":"John Wilson","p":"Chief Technology Officer","b":"https://www.widgetcompany.com/team/johnwilson","l":"johnwilson","t":"jono"},{"n":"Dashna Anand","p":"Chief Marketing Officer","b":"https://www.widgetcompany.com/team/dashnaanand","l":"dashnaanand","t":"dashnaanand"}]]}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"replacePair":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"n":{"rewriteKey":"name"},"p":{"rewriteKey":"position"},"b":{"rewriteKey":"bio"},"l":{"rewriteKey":"linkedin","rewriteValue":"https://www.linkedin.com/in/%self"},"t":{"rewriteKey":"twitter","rewriteValue":"https://www.twitter.com/%self"}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-3b", + "input": {"o":["Widget Company Ltd","Making the best widgets","https://www.widgetcompany.com",[{"em":["Jane Smith","Chief Executive Officer","https://www.widgetcompany.com/team/janesmith","janesmith","janesmithwidgets"]},{"em":["John Wilson","Chief Technology Officer","https://www.widgetcompany.com/team/johnwilson","johnwilson","jono"]},{"em":["Dashna Anand","Chief Marketing Officer","https://www.widgetcompany.com/team/dashnaanand","dashnaanand","dashnaanand"]}]]}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"replacePair":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"em":{"assignKeys":["n","p","b","l","t"],"replacePair":{"name":"%n","position":"%p","bio":"%b","linkedin":"https://www.linkedin.com/in/%l","twitter":"https://www.twitter.com/%t"}}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-4", + "input": {"o":["Widget Company Ltd","Making the best widgets","https://www.widgetcompany.com",[["Jane Smith","Chief Executive Officer","https://www.widgetcompany.com/team/janesmith","janesmith","janesmithwidgets"],["John Wilson","Chief Technology Officer","https://www.widgetcompany.com/team/johnwilson","johnwilson","jono"],["Dashna Anand","Chief Marketing Officer","https://www.widgetcompany.com/team/dashnaanand","dashnaanand","dashnaanand"]]]}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"replacePair":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"e":{"arrayItems":"em"},"em":{"assignKeys":["n","p","b","l","t"],"replacePair":{"name":"%n","position":"%p","bio":"%b","linkedin":"https://www.linkedin.com/in/%l","twitter":"https://www.twitter.com/%t"}}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-5", + "input": {"?":["https://www.widgetcompany.com","team","janesmith","johnwilson","dashnaanand"],"o":["Widget Company Ltd","Making the best widgets","%0",[["Jane Smith","Chief Executive Officer","%0%/%1%/%2","%2","%2%widgets"],["John Wilson","Chief Technology Officer","%0%/%1%/%3","%3","jono"],["Dashna Anand","Chief Marketing Officer","%0%/%1%/%4","%4","%4"]]]}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"replacePair":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"e":{"arrayItems":"em"},"em":{"assignKeys":["n","p","b","l","t"],"replacePair":{"name":"%n","position":"%p","bio":"%b","linkedin":"https://www.linkedin.com/in/%l","twitter":"https://www.twitter.com/%t"}}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "all-step-6", + "input": {"?":["https://www.widgetcompany.com","team","janesmith","johnwilson","dashnaanand"],"o":["Widget Company Ltd","Making the best widgets","%0",[["Jane Smith","%ceo","%0%/%1%/%2","%2","%2%widgets"],["John Wilson","%cto","%0%/%1%/%3","%3","jono"],["Dashna Anand","%cmo","%0%/%1%/%4","%4","%4"]]]}, + "subs": {"ceo":"Chief Executive Officer","cto":"Chief Technology Officer","cmo":"Chief Marketing Officer"}, + "trans": {"o":{"assignKeys":["n","s","w","e"],"replacePair":{"name":"%n","strapline":"%s","website":"%w","employees":"%e"}},"e":{"arrayItems":"em"},"em":{"assignKeys":["n","p","b","l","t"],"replacePair":{"name":"%n","position":"%p","bio":"%b","linkedin":"https://www.linkedin.com/in/%l","twitter":"https://www.twitter.com/%t"}}}, + "output": {"name":"Widget Company Ltd","strapline":"Making the best widgets","website":"https://www.widgetcompany.com","employees":[{"name":"Jane Smith","position":"Chief Executive Officer","bio":"https://www.widgetcompany.com/team/janesmith","linkedin":"https://www.linkedin.com/in/janesmith","twitter":"https://www.twitter.com/janesmithwidgets"},{"name":"John Wilson","position":"Chief Technology Officer","bio":"https://www.widgetcompany.com/team/johnwilson","linkedin":"https://www.linkedin.com/in/johnwilson","twitter":"https://www.twitter.com/jono"},{"name":"Dashna Anand","position":"Chief Marketing Officer","bio":"https://www.widgetcompany.com/team/dashnaanand","linkedin":"https://www.linkedin.com/in/dashnaanand","twitter":"https://www.twitter.com/dashnaanand"}]} + }, { + "id": "NestedPassThrough-1", + "desc": "Unpacker can pass-through nested objects", + "input": {"this":{"that":2,"then":3},"x":"y"}, + "trans": {"this":{"rewriteKey":"foo","that":{"rewriteKey":"bar"}}}, + "output": {"foo":{"bar":2,"then":3},"x":"y"} + }, { + "id": "NestedPassThrough-2", + "desc": "Unpacker can pass-through nested objects", + "input": {"this":{"that":2,"pass-through":"ok","then":{"the other":{"another":"thing","pass-through":"ok"},"pass-through":"ok"}},"x":"y","pass-through":"ok"}, + "trans": {"this":{"rewriteKey":"Alpha","that":{"rewriteKey":"Beta"},"then":{"rewriteKey":"Gamma","the other":{"rewriteKey":"Delta","another":{"rewriteKey":"Epsilon"}}}}}, + "output": {"Alpha":{"Beta":2,"pass-through":"ok","Gamma":{"Delta":{"Epsilon":"thing","pass-through":"ok"},"pass-through":"ok"}},"x":"y","pass-through":"ok"} + }, { + "id": "Supplementary-1", + "desc": "variable index may include arbitrarily nested maps, arrays, and primitives", + "input": {"?":["string",1,true,{"x":{"y":{"z":"Hello World"}}},[{"x":{"y":{"z":"Brave New World"}}}]],"a":"%0","b":"%1","c":"%2","d":"%3.x.y.z","e":"%4.0.x.y.z"}, + "output": {"a":"string","b":1,"c":true,"d":"Hello World","e":"Brave New World"} + }, { + "id": "Supplementary-2", + "desc": "%-prefixes and suffixes", + "input": {"?":["val"],"a":"%0","b":"%0%","c":"x%0","d":"%0 x","e":"x%0%x","f":"x%0%0%x","g":"x%0%%0%x","h":"x%0%%%%0%x","i":"%0 %%0"}, + "output": {"a":"val","b":"val","c":"xval","d":"val x","e":"xvalx","f":"xval0","g":"xvalvalx","h":"xval%%0%x","i":"val %0"} + }, { + "id": "Supplementary-3", + "desc": "unresolved references", + "input": {"a1":"%0","b1":"%0%","c1":"x%0","d1":"%0 x","e1":"x%0%x","f1":"x%0%0%x","g1":"x%0%%0%x","h1":"x%0%%%%0%x","i1":"%0 %%0","a2":"%x","b2":"%x%","c2":"x%x","d2":"%x x","e2":"x%x%x","f2":"x%x%x%x","g2":"x%x%%x%x","h2":"x%x%%%%x%x","i2":"%x %%x"}, + "output": {"a1":null,"b1":null,"c1":"x","d1":" x","e1":"xx","f1":"x0","g1":"xx","h1":"x%%0%x","i1":" %0","a2":null,"b2":null,"c2":"x","d2":" x","e2":"xx","f2":"xx","g2":"xx","h2":"x%%x%x","i2":" %x"} + }, { + "id": "Supplementary-4", + "desc": "remove pairs when `replacePair` is null", + "input": {"foo":"Hello World"}, + "trans": {"foo":{}}, + "output": {} + }, { + "id": "ObjectUnpacker-01", + "input": {"@n":1,"?":["tesco"],"o":{"n":"Tesco","s":"Every Little Helps","c":[{"fb":"%0"},{"pi":"%0"}]}}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"tesco"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"tesco"}}]}} + }, { + "id": "ObjectUnpacker-02", + "input": {"@n":1,"o":{"n":"NUM Example Co","s":"Example Strapline","c":[{"t":{"d":"Customer Service","v":"+441270123456"}},{"fb":{"v":"examplefacebook"}},{"in":{"v":"exampleinstagram"}},{"tw":{"v":"exampletwitter"}}]}}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"object_display_name":"Organisation","name":"NUM Example Co","slogan":"Example Strapline","contacts":[{"telephone":{"object_display_name":"Telephone","description_default":"Call","description":"Customer Service","prefix":"tel:","method_type":"core","value":"+441270123456","hours":null}},{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","description":null,"prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"examplefacebook"}},{"instagram":{"object_display_name":"Instagram","description_default":"View Instagram profile","description":null,"prefix":"https://www.instagram.com/","method_type":"third_party","controller":"instagram.com","value":"exampleinstagram"}},{"twitter":{"object_display_name":"Twitter","description_default":"View Twitter profile","description":null,"prefix":"https://www.twitter.com/","method_type":"third_party","controller":"twitter.com","value":"exampletwitter","value_prefix":"@"}}]}} + }, { + "id": "ObjectUnpacker-03", + "input": {"o":{"n":"Tesco","s":"Every Little Helps","c":["test1","test2"]}}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":["test1","test2"]}} + }, { + "id": "ObjectUnpacker-04", + "input": {"o":{"n":"Tesco","s":"Every Little Helps","c":[{"fb":"test1"},{"pi":"test2"}]}}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"test1"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"test2"}}]}} + }, { + "id": "ObjectUnpacker-05", + "input": {"@n":1,"o":{"n":"Nemo Finders Ltd","id":"832765467","a":["42 West Wallaby Way,","Sydney,","Australia"],"p":"SN1","co":"OZ","e":"bruce@nemo-finders.oz"}}, + "trans": {"@n":{"rewriteValue":"%/compact.@n"},"o":{"rewriteKey":"organisation","assignKeys":["n","id","a","p","co","e"],"rewriteValue":{"name":"%n","identifier":"%id","address":"%a","postcode_zip":"%p","country":"%co","email":"%e","object_display_name":"Organisation"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"name":"Nemo Finders Ltd","identifier":"832765467","address":["42 West Wallaby Way,","Sydney,","Australia"],"postcode_zip":"SN1","country":"OZ","email":"bruce@nemo-finders.oz","object_display_name":"Organisation"}} + }, { + "id": "ObjectUnpacker-06", + "input": {"@n":1,"o":{"n":true,"id":[1],"a":[1,2,3,4,true,false],"p":"SN25 4YF","co":"UK","e":"tony@aosd.co.uk"}}, + "trans": {"@n":{"rewriteValue":"%/compact.@n"},"o":{"rewriteKey":"organisation","assignKeys":["n","id","a","p","co","e"],"rewriteValue":{"name":"%n","identifier":"%id","address":"%a","postcode_zip":"%p","country":"%co","email":"%e","object_display_name":"Organisation"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"name":true,"identifier":[1],"address":[1,2,3,4,true,false],"postcode_zip":"SN25 4YF","country":"UK","email":"tony@aosd.co.uk","object_display_name":"Organisation"}} + }, { + "id": "ObjectUnpacker-07", + "input": {"o":"tesco"}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"organisation":{"name":"tesco","object_display_name":"Organisation"}} + }, { + "id": "ObjectUnpacker-08", + "input": {"p":{"f":"John","l":"Smith","a":20}}, + "trans": {"p":{"rewriteKey":"person","rewriteValue":{"first":"%/compact.p.f","last":"%/compact.p.l","age":"%/compact.p.a"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"person":{"first":"John","last":"Smith","age":20}} + }, { + "id": "ObjectUnpacker-09", + "input": {"p":["John","Smith",20]}, + "trans": {"p":{"rewriteKey":"person","assignKeys":["first","last","age"]}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"person":{"first":"John","last":"Smith","age":20}} + }, { + "id": "ObjectUnpacker-10", + "input": {"p":["John","Smith",20]}, + "trans": {"p":{"rewriteKey":"person","assignKeys":["first","last","age"],"rewriteValue":{"name":{"first":"%first","last":"%last"},"age":"%age","link":"https://www.example.com/team/%first%%last%"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"person":{"name":{"first":"John","last":"Smith"},"age":20,"link":"https://www.example.com/team/JohnSmith"}} + }, { + "id": "ObjectUnpacker-10.5", + "input": {"@n":1,"o":{"c":[{"l":{"d":"Hotel Reception","@L":"/hotel"}},{"l":{"d":"Restaurant","@L":"/restaurant"}},{"t":{"d":"Call Us On","v":"+441270625283","h":{"tz":"LON","av":["wd@7-23","6@7-0","7@8-23"]}}},{"fb":{"v":"The-Crown-Hotel-Bar-Grill-195748670483812","d":"Follow Us"}},{"tw":{"v":"Crown_Hotel","d":"Follow Us"}},{"u":{"v":"crownhotelnantwich.com","d":"Visit Website"}},{"fx":{"v":"01270 628047"}},{"em":{"v":"info@crownhotelnantwich.com","d":"Send an email"}},{"a":{"al":["High Street","Nantwich","Cheshire","CW5 5AS"],"d":"Find Us"}}],"n":"The Crown Hotel Bar and Grill Nantwich"}}, + "trans_file": "schema-map.json", + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"object_display_name":"Organisation","name":"The Crown Hotel Bar and Grill Nantwich","slogan":null,"contacts":[{"link":{"@L":"/hotel","description":"Hotel Reception","name":null,"object_display_name":"Link"}},{"link":{"@L":"/restaurant","description":"Restaurant","name":null,"object_display_name":"Link"}},{"telephone":{"object_display_name":"Telephone","description_default":"Call","description":"Call Us On","prefix":"tel:","method_type":"core","value":"+441270625283","hours":{"time_zone_location":"LON","available":["wd@7-23","6@7-0","7@8-23"]}}},{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","description":"Follow Us","prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"The-Crown-Hotel-Bar-Grill-195748670483812"}},{"twitter":{"object_display_name":"Twitter","description_default":"View Twitter profile","description":"Follow Us","prefix":"https://www.twitter.com/","method_type":"third_party","controller":"twitter.com","value":"Crown_Hotel","value_prefix":"@"}},{"url":{"object_display_name":"Web URL","description_default":"Click","description":"Visit Website","prefix":"https://","method_type":"core","value":"crownhotelnantwich.com"}},{"fax":{"object_display_name":"Fax","description_default":"Send a fax","description":null,"prefix":"tel:","method_type":"core","value":"01270 628047"}},{"email":{"object_display_name":"Email","description_default":"Send an email","description":"Send an email","prefix":"mailto:","method_type":"core","value":"info@crownhotelnantwich.com"}},{"address":{"description":"Find Us","description_default":"View Address","lines":["High Street","Nantwich","Cheshire","CW5 5AS"],"postcode":null,"country":null,"method_type":"core","object_display_name":"Address","prefix":""}}]}} + }, { + "id": "ObjectUnpacker-11", + "input": {"@n":1,"o":{"n":"NUM Technology Ltd","id":10097965,"a":["39th Floor","One Canada Square","Canary Wharf","London"],"pz":"E14 5AB","co":"GB","t":"+441270123456","e":"domains@num.uk","c":[{"AD":{"n":"Mr Admin Contact","a":["Admin Department","39th Floor","One Canada Square","Canary Wharf","London"],"pz":"E14 5AB","co":"GB","e":"domains-admin@num.uk","t":"+441270654321"}},{"TC":{"n":"Ms Tech Contact","a":["Tech Team","39th Floor","One Canada Square","Canary Wharf","London"],"pz":"E14 5AB","t":"+441270111111","e":"domains-tech@num.uk","co":"GB"}},{"SC":{"n":"Ms Security Contact","a":["Tech Team","39th Floor","One Canada Square","Canary Wharf","London"],"pz":"E14 5AB","co":"GB","t":"+441270222222","e":"domains-security@num.uk"}}]}}, + "trans": {"@n":{"rewriteValue":"%/compact.@n"},"o":{"rewriteKey":"organisation","assignKeys":["n","id","a","pz","co","t","e","c"],"rewriteValue":{"name":"%n","identifier":"%id","address":"%a","postcode_zip":"%pz","country":"%co","telephone":"%t","email":"%e","contacts":"%c","object_display_name":"Organisation"}},"AD":{"rewriteKey":"AD","assignKeys":["n","a","pz","co","e","t"],"rewriteValue":{"name":"%n","address":"%a","postcode_zip":"%pz","country":"%co","telephone":"%t","email":"%e"}},"TC":{"rewriteKey":"TC","assignKeys":["n","a","pz","co","e","t"],"rewriteValue":{"name":"%n","address":"%a","postcode_zip":"%pz","country":"%co","telephone":"%t","email":"%e"}},"SC":{"rewriteKey":"SC","assignKeys":["n","a","pz","co","e","t"],"rewriteValue":{"name":"%n","address":"%a","postcode_zip":"%pz","country":"%co","telephone":"%t","email":"%e"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"organisation":{"name":"NUM Technology Ltd","identifier":10097965,"address":["39th Floor","One Canada Square","Canary Wharf","London"],"postcode_zip":"E14 5AB","country":"GB","telephone":"+441270123456","email":"domains@num.uk","contacts":[{"AD":{"name":"Mr Admin Contact","address":["Admin Department","39th Floor","One Canada Square","Canary Wharf","London"],"postcode_zip":"E14 5AB","country":"GB","telephone":"+441270654321","email":"domains-admin@num.uk"}},{"TC":{"name":"Ms Tech Contact","address":["Tech Team","39th Floor","One Canada Square","Canary Wharf","London"],"postcode_zip":"E14 5AB","country":"GB","telephone":"+441270111111","email":"domains-tech@num.uk"}},{"SC":{"name":"Ms Security Contact","address":["Tech Team","39th Floor","One Canada Square","Canary Wharf","London"],"postcode_zip":"E14 5AB","country":"GB","telephone":"+441270222222","email":"domains-security@num.uk"}}],"object_display_name":"Organisation"}} + }, { + "id": "ObjectUnpacker-12", + "input": {"@n":1,"i":[{"n":"NUM Logo","t":"logo","v":[["https://www.logos.uk/num/num.uk-100.png","image/png",100,100],["https://www.logos.uk/num/num.uk-250.png","image/png",250,250],["https://www.logos.uk/num/num.uk-500.png","image/png",500,500]]},{"n":"NUM Strapline","t":"splash","v":[["https://www.logos.uk/num/num.uk-strapline.png","image/png",250,250],["https://www.logos.uk/num/num.uk-strapline-500.png","image/png",500,500],["https://www.logos.uk/num/num.uk-strapline-1000.png","image/png",1000,1000]]}]}, + "trans": {"@n":{"rewriteValue":"%/compact.@n"},"i":{"rewriteKey":"images","arrayItems":"image"},"image":{"assignKeys":["n","t","v"],"replacePair":{"name":"%n","type":"%t","variants":"%v"}},"v":{"arrayItems":"variant"},"variant":{"assignKeys":["u","m","w","h"],"replacePair":{"url":"%u","mime":"%m","width":"%w","height":"%h"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"images":[{"name":"NUM Logo","type":"logo","variants":[{"url":"https://www.logos.uk/num/num.uk-100.png","mime":"image/png","width":100,"height":100},{"url":"https://www.logos.uk/num/num.uk-250.png","mime":"image/png","width":250,"height":250},{"url":"https://www.logos.uk/num/num.uk-500.png","mime":"image/png","width":500,"height":500}]},{"name":"NUM Strapline","type":"splash","variants":[{"url":"https://www.logos.uk/num/num.uk-strapline.png","mime":"image/png","width":250,"height":250},{"url":"https://www.logos.uk/num/num.uk-strapline-500.png","mime":"image/png","width":500,"height":500},{"url":"https://www.logos.uk/num/num.uk-strapline-1000.png","mime":"image/png","width":1000,"height":1000}]}]} + }, { + "id": "ObjectUnpacker-13", + "input": {"@n":1,"c":[["be165e855fc34cee0cdfd1b68921152132ee9191",["*"]],["8bfa323c01e3a7f0e90f7685d1b0188b7c8db37e",["numserver.com"]],["a548a3b5f8920fac10530f63da8ce63b67cba768",["m","e","s","c"]]],"b":true}, + "trans": {"@n":{"rewriteValue":"%/compact.@n"},"b":{"rewriteKey":"branches"},"c":{"rewriteKey":"custodians","arrayItems":"custodian"},"custodian":{"assignKeys":["h","perms"],"replacePair":{"hash":"%h","permissions":"%perms"}}}, + "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, + "output": {"@n":1,"custodians":[{"hash":"be165e855fc34cee0cdfd1b68921152132ee9191","permissions":["*"]},{"hash":"8bfa323c01e3a7f0e90f7685d1b0188b7c8db37e","permissions":["numserver.com"]},{"hash":"a548a3b5f8920fac10530f63da8ce63b67cba768","permissions":["m","e","s","c"]}],"branches":true} } ] \ No newline at end of file diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 2ff05e4..1ded478 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -8,19 +8,32 @@ import ( ) type UnpackerTest struct { - ID string - Input json.RawMessage - Subs Substitution - Trans map[string]Transform - Output json.RawMessage - Skip bool + ID string + Input json.RawMessage + Subs Substitution + Trans map[string]Transform + TransFile string `json:"trans_file"` + Output json.RawMessage + Skip bool } func (test *UnpackerTest) Run(t *testing.T) { + must := func(err error) { + if err != nil { + t.Error(err) + t.FailNow() + } + } if testing.Short() && test.Skip { t.Skip("manually disabled test") } u := &Unpacker{} + if len(test.TransFile) != 0 { + bits, err := ioutil.ReadFile(test.TransFile) + must(err) + test.Trans, err = ParseTransforms(bits) + must(err) + } if len(test.Trans) != 0 { u.Transforms = test.Trans // fmt.Printf("Got Transforms: %v\n", u.Transforms) @@ -52,7 +65,18 @@ func (test UnpackerTest) out() []byte { } var skip = map[string]bool{ - "Unpacker-": true, + "all-step-3a-1": true, // assign key + replace pair? + "all-step-3a-2": true, // assign key + replace... + "Supplementary-1": true, // unexpected bool + "Supplementary-2": true, // % escapes? + "Supplementary-3": true, // unresolved refs + "Supplementary-4": true, // implicit replacePair: null + "ObjectUnpacker-01": true, + "ObjectUnpacker-04": true, + "ObjectUnpacker-06": true, // unexpected bool + "ObjectUnpacker-07": true, + "ObjectUnpacker-10": true, + "ObjectUnpacker-13": true, // unexpected bool } func TestUnpacker(t *testing.T) { From 56782977ec5d37f845c4e9df392f2e3373226a5b Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Tue, 25 May 2021 22:07:16 +0000 Subject: [PATCH 20/27] fix bool --- unpacker/unpacker.go | 2 +- unpacker/unpacker_test.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 44dd2dd..bd206bc 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -211,7 +211,7 @@ func (state unpackState) value() interface{} { case string: // fmt.Printf("Got a string value: %q\n", token) return state.string(v, state.fullMem) - case float64: + case float64, bool: // fmt.Printf("Got an int value: %f\n", token) return v case nil: diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 1ded478..179f8f9 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -67,16 +67,13 @@ func (test UnpackerTest) out() []byte { var skip = map[string]bool{ "all-step-3a-1": true, // assign key + replace pair? "all-step-3a-2": true, // assign key + replace... - "Supplementary-1": true, // unexpected bool "Supplementary-2": true, // % escapes? "Supplementary-3": true, // unresolved refs "Supplementary-4": true, // implicit replacePair: null "ObjectUnpacker-01": true, "ObjectUnpacker-04": true, - "ObjectUnpacker-06": true, // unexpected bool "ObjectUnpacker-07": true, - "ObjectUnpacker-10": true, - "ObjectUnpacker-13": true, // unexpected bool + "ObjectUnpacker-10": true, // assign key + rewrite value } func TestUnpacker(t *testing.T) { From d6f6df3ca21e5e8b118cbe63a709264b4691f7cb Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Tue, 25 May 2021 22:32:52 +0000 Subject: [PATCH 21/27] fix resolve with context --- unpacker/unpacker.go | 42 +++++++++++++++++++-------------------- unpacker/unpacker_test.go | 1 - 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index bd206bc..83790ff 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -210,7 +210,7 @@ func (state unpackState) value() interface{} { } case string: // fmt.Printf("Got a string value: %q\n", token) - return state.string(v, state.fullMem) + return resolveString(v, state.fullMem) case float64, bool: // fmt.Printf("Got an int value: %f\n", token) return v @@ -271,7 +271,22 @@ func (state unpackState) array() []interface{} { return o } -func (state unpackState) string(in string, subz map[string]interface{}) interface{} { +func resolveObject(o interface{}, ctx map[string]interface{}) interface{} { + switch v := o.(type) { + case string: + return resolveString(v, ctx) + case map[string]interface{}: + u := make(map[string]interface{}, len(v)) // duplicate in case of modifications + for k, w := range v { + u[k] = resolveObject(w, ctx) + } + return u + default: + return o + } +} + +func resolveString(in string, subz map[string]interface{}) interface{} { // NOTE: state is not used, but is left as a potential memory optimization // (array of contexts or logic to switch between allowing viariadic or not) for k, v := range subz { @@ -380,7 +395,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu if trans.ReturnNull { return // if replacePair is null, the key should not be assigned } else if trans.ReturnStr != "" { - obj := state.string(trans.ReturnStr, ctx) + obj := resolveString(trans.ReturnStr, ctx) if m, ok := obj.(map[string]interface{}); ok { for k, v := range m { dest[k] = v @@ -390,11 +405,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu panic(fmt.Sprintf("Got unexpected ReturnString type: %T", obj)) } else if trans.Return != nil { for k, v := range trans.Return { - // TODO: smarter string resolves - if s, ok := v.(string); ok { - v = state.string(s, ctx) - } - dest[k] = v + dest[k] = resolveObject(v, ctx) } return } @@ -404,20 +415,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu dest[key] = nil // if rewriteValue is null, assign value to null return } else if trans.Rewrite != nil { - if m, ok := trans.Rewrite.(map[string]interface{}); ok { - n := map[string]interface{}{} // ensure we don't duplicate the object - for k, v := range m { - if s, ok := v.(string); ok { - v = state.string(s, ctx) - } - n[k] = v - } - value = n - } else if s, ok := trans.Rewrite.(string); ok { - value = state.string(s, ctx) - } else { - panic(fmt.Sprintf("got an unexpected rewriteValue type: %T", trans.Rewrite)) - } + value = resolveObject(trans.Rewrite, ctx) } // 4: rewriteKey, replace key if need be diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 179f8f9..e6984d6 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -73,7 +73,6 @@ var skip = map[string]bool{ "ObjectUnpacker-01": true, "ObjectUnpacker-04": true, "ObjectUnpacker-07": true, - "ObjectUnpacker-10": true, // assign key + rewrite value } func TestUnpacker(t *testing.T) { From 6d011d63ef1e7e8eec74b452a605ee1275b77ddf Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Tue, 25 May 2021 22:47:12 +0000 Subject: [PATCH 22/27] fix replace with key rename --- unpacker/unpacker.go | 1 + unpacker/unpacker_test.go | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 83790ff..1315280 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -374,6 +374,7 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu } newValue := make(map[string]interface{}, len(list)) for i, v := range list { + newValue[trans.Assign[i]] = v // assign for future ctx logic state.transform(newValue, trans.Assign[i], v) // recurse to assign key/values } value = newValue diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index e6984d6..f5b7b94 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -65,8 +65,6 @@ func (test UnpackerTest) out() []byte { } var skip = map[string]bool{ - "all-step-3a-1": true, // assign key + replace pair? - "all-step-3a-2": true, // assign key + replace... "Supplementary-2": true, // % escapes? "Supplementary-3": true, // unresolved refs "Supplementary-4": true, // implicit replacePair: null From 4e3fb5dabff7f4e8ed6a213f37fbc7d109ccdb31 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Wed, 26 May 2021 03:35:57 +0000 Subject: [PATCH 23/27] default replacePair: null if nothing is set --- unpacker/unpacker.go | 16 ++++++++++++++++ unpacker/unpacker_test.go | 7 +++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 1315280..8fa01ea 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -36,11 +36,13 @@ type Transform struct { func (t *Transform) UnmarshalJSON(bits []byte) error { // TODO: emit errors if the types are wrong + var flag bool trans := make(map[string]interface{}) // todo: map string to json.RawMessage? if err := json.Unmarshal(bits, &trans); err != nil { return err } if assign, ok := trans["assignKeys"].([]interface{}); ok { + flag = true t.Assign = make([]string, 0, len(assign)) for _, a := range assign { t.Assign = append(t.Assign, a.(string)) @@ -48,14 +50,17 @@ func (t *Transform) UnmarshalJSON(bits []byte) error { delete(trans, "assignKeys") } if items, ok := trans["arrayItems"].(string); ok { + flag = true t.Items = items delete(trans, "arrayItems") } if key, ok := trans["rewriteKey"].(string); ok { + flag = true t.Key = key delete(trans, "rewriteKey") } if replace, ok := trans["replacePair"]; ok { + flag = true if replace == nil { t.ReturnNull = true } else if str, ok := replace.(string); ok { @@ -66,6 +71,7 @@ func (t *Transform) UnmarshalJSON(bits []byte) error { delete(trans, "replacePair") } if rewrite, ok := trans["rewriteValue"]; ok { + flag = true if rewrite == nil { t.RewriteNull = true } else { @@ -74,6 +80,7 @@ func (t *Transform) UnmarshalJSON(bits []byte) error { delete(trans, "rewriteValue") } if len(trans) > 0 { + flag = true t.Nesting = make(map[string]Transform) bits, err := json.Marshal(trans) if err != nil { @@ -81,6 +88,9 @@ func (t *Transform) UnmarshalJSON(bits []byte) error { } return json.Unmarshal(bits, &t.Nesting) } + if !flag { + t.ReturnNull = true // if nothing else is set, assume returnNull is true + } return nil } @@ -385,6 +395,12 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu m[k] = nil } } + } else if len(trans.Assign) != 0 { + // iff other, assume the value is the firs t key set {fb: "asdf"} + value = map[string]interface{}{ + trans.Assign[0]: value, + } + // TODO: treat the unset keys as deletions } // 1.5: update ctx as necessary diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index f5b7b94..15c6ac0 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -67,10 +67,9 @@ func (test UnpackerTest) out() []byte { var skip = map[string]bool{ "Supplementary-2": true, // % escapes? "Supplementary-3": true, // unresolved refs - "Supplementary-4": true, // implicit replacePair: null - "ObjectUnpacker-01": true, - "ObjectUnpacker-04": true, - "ObjectUnpacker-07": true, + "ObjectUnpacker-01": true, // assignKeys with single value, don't set subsequent properties that only reference unset fields? + "ObjectUnpacker-04": true, // same + "ObjectUnpacker-07": true, // same } func TestUnpacker(t *testing.T) { From cc506aef6e3fd1918d27b2131c6aceebdb5a12e6 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Thu, 27 May 2021 20:24:38 +0000 Subject: [PATCH 24/27] always assign null to unset keys --- unpacker/tests.json | 12 +++++++++--- unpacker/unpacker.go | 7 +++++-- unpacker/unpacker_test.go | 7 ++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/unpacker/tests.json b/unpacker/tests.json index bb47d6f..9c7fb7e 100644 --- a/unpacker/tests.json +++ b/unpacker/tests.json @@ -107,6 +107,12 @@ "input": {"a":["alpha","bravo","charlie"]}, "trans": {"a":{"assignKeys":["a","b","c"],"rewriteValue":"%c"}}, "output": {"a":"charlie"} + }, { + "id": "Unpacker-19", + "desc": "ch11442", + "input": {"assign":"a","replace":{"a":"a","b":null},"both":"a"}, + "trans": {"assign":{"assignKeys":["assign_a","assign_b"]},"replace":{"replacePair":{"replace_a":"%a","replace_b":"%b"}},"both":{"assignKeys":["both_a","both_b"],"replacePair":{"both_a":"%both_a","both_b":"%both_b"}}}, + "output": {"assign":{"assign_a":"a","assign_b":null},"replace_a":"a","replace_b":null,"both_a":"a","both_b":null} }, { "id": "Nested-01", "desc": "It should be possible to nest unpacker specs", @@ -1495,7 +1501,7 @@ "input": {"@n":1,"?":["tesco"],"o":{"n":"Tesco","s":"Every Little Helps","c":[{"fb":"%0"},{"pi":"%0"}]}}, "trans_file": "schema-map.json", "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, - "output": {"@n":1,"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"tesco"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"tesco"}}]}} + "output": {"@n":1,"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","description":null,"prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"tesco"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","description":null,"prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"tesco"}}]}} }, { "id": "ObjectUnpacker-02", "input": {"@n":1,"o":{"n":"NUM Example Co","s":"Example Strapline","c":[{"t":{"d":"Customer Service","v":"+441270123456"}},{"fb":{"v":"examplefacebook"}},{"in":{"v":"exampleinstagram"}},{"tw":{"v":"exampletwitter"}}]}}, @@ -1513,7 +1519,7 @@ "input": {"o":{"n":"Tesco","s":"Every Little Helps","c":[{"fb":"test1"},{"pi":"test2"}]}}, "trans_file": "schema-map.json", "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, - "output": {"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"test1"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"test2"}}]}} + "output": {"organisation":{"object_display_name":"Organisation","name":"Tesco","slogan":"Every Little Helps","contacts":[{"facebook":{"object_display_name":"Facebook","description_default":"View Facebook profile","description":null,"prefix":"https://www.facebook.com/","method_type":"third_party","controller":"facebook.com","value":"test1"}},{"pinterest":{"object_display_name":"Pinterest","description_default":"View Pinterest board","description":null,"prefix":"https://www.pinterest.com/","method_type":"third_party","controller":"pinterest.com","value":"test2"}}]}} }, { "id": "ObjectUnpacker-05", "input": {"@n":1,"o":{"n":"Nemo Finders Ltd","id":"832765467","a":["42 West Wallaby Way,","Sydney,","Australia"],"p":"SN1","co":"OZ","e":"bruce@nemo-finders.oz"}}, @@ -1531,7 +1537,7 @@ "input": {"o":"tesco"}, "trans_file": "schema-map.json", "subs": {"locale":{"o":{"name":"Organisation"},"fb":{"name":"Facebook","default":"View Facebook profile"},"tw":{"name":"Twitter","default":"View Twitter profile"},"pi":{"name":"Pinterest","default":"View Pinterest board"},"in":{"name":"Instagram","default":"View Instagram profile"},"l":{"name":"Link","default":"Follow Link"},"t":{"name":"Telephone","default":"Call"},"u":{"name":"Web URL","default":"Click"},"fx":{"name":"Fax","default":"Send a fax"},"em":{"name":"Email","default":"Send an email"},"a":{"name":"Address","default":"View Address"}}}, - "output": {"organisation":{"name":"tesco","object_display_name":"Organisation"}} + "output": {"organisation":{"object_display_name":"Organisation","name":"tesco","slogan":null,"contacts":null}} }, { "id": "ObjectUnpacker-08", "input": {"p":{"f":"John","l":"Smith","a":20}}, diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 8fa01ea..1404100 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -396,11 +396,14 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu } } } else if len(trans.Assign) != 0 { - // iff other, assume the value is the firs t key set {fb: "asdf"} + // iff other, assume the value is the first key set {fb: "asdf"} value = map[string]interface{}{ trans.Assign[0]: value, } - // TODO: treat the unset keys as deletions + // Assign unset keys to be nil + for _, v := range trans.Assign[1:] { + value.(map[string]interface{})[v] = nil + } } // 1.5: update ctx as necessary diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 15c6ac0..57d5afa 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -65,11 +65,8 @@ func (test UnpackerTest) out() []byte { } var skip = map[string]bool{ - "Supplementary-2": true, // % escapes? - "Supplementary-3": true, // unresolved refs - "ObjectUnpacker-01": true, // assignKeys with single value, don't set subsequent properties that only reference unset fields? - "ObjectUnpacker-04": true, // same - "ObjectUnpacker-07": true, // same + "Supplementary-2": true, // % escapes? (stronger string parsing) + "Supplementary-3": true, // unresolved refs (stronger string parsing) } func TestUnpacker(t *testing.T) { From fed86d19e5a9f58b01fff8c009c6d303a25470b9 Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Fri, 28 May 2021 12:38:12 +0000 Subject: [PATCH 25/27] Fix string supplementary tests to match results Note: Supplementary-2 and 3 needed manual updating to match my expectations. Still waiting discussion in the NumTechnology Slack, but just getting things moving for now. --- unpacker/tests.json | 42 +++++++++++++++++++++++--- unpacker/unpacker.go | 63 ++++++++++++++++++++++++++------------- unpacker/unpacker_test.go | 8 ++--- 3 files changed, 85 insertions(+), 28 deletions(-) diff --git a/unpacker/tests.json b/unpacker/tests.json index 9c7fb7e..33a2e22 100644 --- a/unpacker/tests.json +++ b/unpacker/tests.json @@ -1482,14 +1482,14 @@ "output": {"a":"string","b":1,"c":true,"d":"Hello World","e":"Brave New World"} }, { "id": "Supplementary-2", - "desc": "%-prefixes and suffixes", + "desc": "%-prefixes and suffixes (NOTE: h expected is different from NUMTechnology/object-unpacker)", "input": {"?":["val"],"a":"%0","b":"%0%","c":"x%0","d":"%0 x","e":"x%0%x","f":"x%0%0%x","g":"x%0%%0%x","h":"x%0%%%%0%x","i":"%0 %%0"}, - "output": {"a":"val","b":"val","c":"xval","d":"val x","e":"xvalx","f":"xval0","g":"xvalvalx","h":"xval%%0%x","i":"val %0"} + "output": {"a":"val","b":"val","c":"xval","d":"val x","e":"xvalx","f":"xval0","g":"xvalvalx","h":"xval%valx","i":"val %0"} }, { "id": "Supplementary-3", - "desc": "unresolved references", + "desc": "unresolved references (NOTE: h1 and h2 expected is different from NUMTechnology/object-unpacker)", "input": {"a1":"%0","b1":"%0%","c1":"x%0","d1":"%0 x","e1":"x%0%x","f1":"x%0%0%x","g1":"x%0%%0%x","h1":"x%0%%%%0%x","i1":"%0 %%0","a2":"%x","b2":"%x%","c2":"x%x","d2":"%x x","e2":"x%x%x","f2":"x%x%x%x","g2":"x%x%%x%x","h2":"x%x%%%%x%x","i2":"%x %%x"}, - "output": {"a1":null,"b1":null,"c1":"x","d1":" x","e1":"xx","f1":"x0","g1":"xx","h1":"x%%0%x","i1":" %0","a2":null,"b2":null,"c2":"x","d2":" x","e2":"xx","f2":"xx","g2":"xx","h2":"x%%x%x","i2":" %x"} + "output": {"a1":null,"b1":null,"c1":"x","d1":" x","e1":"xx","f1":"x0","g1":"xx","h1":"x%x","i1":" %0","a2":null,"b2":null,"c2":"x","d2":" x","e2":"xx","f2":"xx","g2":"xx","h2":"x%x","i2":" %x"} }, { "id": "Supplementary-4", "desc": "remove pairs when `replacePair` is null", @@ -1497,6 +1497,40 @@ "trans": {"foo":{}}, "output": {} }, { + "id": "Supplementary-n8", + "desc": "test does not exist in NumTechnology/object-unpacker", + "input": { + "?": [ + 0, + null, + [], + {}, + false + ], + "a": "%0", + "b": "==%0%==", + "c": "%1", + "d": "==%1%==", + "e": "%2", + "f": "==%2%==", + "g": "%3", + "h": "==%3%==", + "i": "%4", + "j": "==%4%==" + }, + "output": { + "a": 0, + "b": "==0==", + "c": null, + "d": "====", + "e": [], + "f": "====", + "g": {}, + "h": "====", + "i": false, + "j": "==false==" + } + }, { "id": "ObjectUnpacker-01", "input": {"@n":1,"?":["tesco"],"o":{"n":"Tesco","s":"Every Little Helps","c":[{"fb":"%0"},{"pi":"%0"}]}}, "trans_file": "schema-map.json", diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 1404100..0c02e1a 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -297,28 +297,51 @@ func resolveObject(o interface{}, ctx map[string]interface{}) interface{} { } func resolveString(in string, subz map[string]interface{}) interface{} { - // NOTE: state is not used, but is left as a potential memory optimization - // (array of contexts or logic to switch between allowing viariadic or not) - for k, v := range subz { - key := "%" + k - if in == key || in == key+"%" { - return v + first := strings.IndexByte(in, '%') + if len(in) == 0 || first == -1 { + return in // noop if nothing can be a reference here + } + + // Check if the whole things is a reference real quick + if clean := strings.TrimSuffix(in[1:], "%"); first == 0 && !strings.ContainsAny(clean, " %") { + return subz[clean] + } + + // Loop through one % at a time, searching for escape characters and closing percent symbols + for first >= 0 && first < len(in) { + + // find the next excape character and set the slice so in[first:next] is the string that needs replaced + next := strings.IndexAny(in[first+1:], " %") + if next < 0 { + next = len(in) + } else if in[first+1+next] == '%' { + next += first + 2 + } else { + next += first + 1 } - if s, ok := v.(string); ok { - if strings.HasSuffix(in, key) { - in = strings.TrimSuffix(in, key) + s - } - if strings.Contains(in, key+" ") { - in = strings.Replace(in, key+" ", s+" ", -1) - } - if strings.Contains(in, key+"%") { - in = strings.Replace(in, key+"%", s, -1) - } + + // See what the replacement string looks like + word := strings.TrimSuffix(in[first+1:next], "%") + if len(word) == 0 { + word = "%" + } else if x, ok := subz[word]; ok { + word = x.(string) // TODO: more type assertions + } else { + word = "" + } + + // replace the section in the resulting string + in = in[:first] + word + in[next:] + next = first + len(word) + + // what's next? + if next > len(in)-1 { + break + } + first = strings.IndexByte(in[next:], '%') + if first >= 0 { + first += next } - } - // assume some part of the full thing was a key (likely a bug, need a beter way to do this) - if strings.HasPrefix(in, "%") { - return nil } return in } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 57d5afa..cebbdf9 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -64,10 +64,7 @@ func (test UnpackerTest) out() []byte { return bits } -var skip = map[string]bool{ - "Supplementary-2": true, // % escapes? (stronger string parsing) - "Supplementary-3": true, // unresolved refs (stronger string parsing) -} +var skip = map[string]bool{} func TestUnpacker(t *testing.T) { jsonTests, err := ioutil.ReadFile("tests.json") @@ -79,6 +76,9 @@ func TestUnpacker(t *testing.T) { t.Fatal("Unable to deserialize tests: " + err.Error()) } for _, test := range tests { + if test.ID == "Supplementary-n8" { + continue + } test.Skip = skip[test.ID] t.Run(test.ID, test.Run) } From 987fc764dc69b9cfa952cfdcf4cf46a89588a02a Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Fri, 28 May 2021 12:55:39 +0000 Subject: [PATCH 26/27] Attempting to corce non-string objects to strings --- unpacker/unpacker.go | 19 ++++++++++++++++++- unpacker/unpacker_test.go | 3 --- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 0c02e1a..7a44902 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -325,7 +325,24 @@ func resolveString(in string, subz map[string]interface{}) interface{} { if len(word) == 0 { word = "%" } else if x, ok := subz[word]; ok { - word = x.(string) // TODO: more type assertions + switch y := x.(type) { + case string: + word = y + case float64: + word = strconv.FormatFloat(y, 'G', -1, 64) + case bool: + if y { + word = "true" + } else { + word = "false" + } + case nil: + word = "" + case []interface{}, map[string]interface{}: + word = "" // NOTE: no idea how we want to conver these typesto strings + default: + panic(fmt.Sprintf("resolveString: unexpected type %T", x)) + } } else { word = "" } diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index cebbdf9..6f0a4f8 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -76,9 +76,6 @@ func TestUnpacker(t *testing.T) { t.Fatal("Unable to deserialize tests: " + err.Error()) } for _, test := range tests { - if test.ID == "Supplementary-n8" { - continue - } test.Skip = skip[test.ID] t.Run(test.ID, test.Run) } From 2042a5a490bd1af510bd5c6bffc60401afa2a42f Mon Sep 17 00:00:00 2001 From: Nate Woods Date: Sat, 29 May 2021 04:39:44 +0000 Subject: [PATCH 27/27] drop comments / cleanup --- unpacker/unpacker.go | 11 +---------- unpacker/unpacker_test.go | 6 +++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/unpacker/unpacker.go b/unpacker/unpacker.go index 7a44902..ab9bf7b 100644 --- a/unpacker/unpacker.go +++ b/unpacker/unpacker.go @@ -186,7 +186,6 @@ func (u Unpacker) Unpack(source []byte) ([]byte, error) { augment("/subs.", u.Subs, state.noVariMem) augment("/compact.", compact, state.fullMem) augment("/compact.", compact, state.noVariMem) - // fmt.Printf("Values: %#v\n", state.mem) // pass two, process the data state.dec = json.NewDecoder(bytes.NewReader(source)) @@ -209,7 +208,6 @@ func (state unpackState) value() interface{} { } switch v := token.(type) { case json.Delim: - // fmt.Printf("Got a delimiter: %v\n", token) switch v.String() { case "{": return state.object() @@ -219,10 +217,8 @@ func (state unpackState) value() interface{} { panic("Unknown delim string: " + v.String()) } case string: - // fmt.Printf("Got a string value: %q\n", token) return resolveString(v, state.fullMem) case float64, bool: - // fmt.Printf("Got an int value: %f\n", token) return v case nil: return nil @@ -331,11 +327,7 @@ func resolveString(in string, subz map[string]interface{}) interface{} { case float64: word = strconv.FormatFloat(y, 'G', -1, 64) case bool: - if y { - word = "true" - } else { - word = "false" - } + word = strconv.FormatBool(y) case nil: word = "" case []interface{}, map[string]interface{}: @@ -449,7 +441,6 @@ func (state unpackState) transform(dest map[string]interface{}, key string, valu // 1.5: update ctx as necessary ctx["self"] = value augment("", value, ctx) - // fmt.Printf("Got context: %v\n", ctx) // 2: replacePair and exit if trans.ReturnNull { diff --git a/unpacker/unpacker_test.go b/unpacker/unpacker_test.go index 6f0a4f8..a6a5358 100644 --- a/unpacker/unpacker_test.go +++ b/unpacker/unpacker_test.go @@ -3,7 +3,7 @@ package unpacker import ( "bytes" "encoding/json" - "io/ioutil" + "os" "testing" ) @@ -29,7 +29,7 @@ func (test *UnpackerTest) Run(t *testing.T) { } u := &Unpacker{} if len(test.TransFile) != 0 { - bits, err := ioutil.ReadFile(test.TransFile) + bits, err := os.ReadFile(test.TransFile) must(err) test.Trans, err = ParseTransforms(bits) must(err) @@ -67,7 +67,7 @@ func (test UnpackerTest) out() []byte { var skip = map[string]bool{} func TestUnpacker(t *testing.T) { - jsonTests, err := ioutil.ReadFile("tests.json") + jsonTests, err := os.ReadFile("tests.json") if err != nil { t.Skip("Cannot read base_tests.json (git submodule init): " + err.Error()) }