Skip to content

Commit 5eea8be

Browse files
rsdoielrsdoiel
committed
fix: upgrade to custom JSON handlers
1 parent 9cbb1a9 commit 5eea8be

27 files changed

+312
-253
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,5 @@ bin/*
3737
# Project ignores
3838
*.json
3939
testout
40+
temp.*
41+
temp?.*

‎CITATION.cff

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ maintainers:
1616
orcid: "https://orcid.org/0000-0003-0900-6903"
1717

1818
repository-code: "https://github.com/caltechlibrary/datatools"
19-
version: 1.2.3
19+
version: 1.2.4
2020
license-url: "https://data.caltech.edu/license"
2121
keywords: [ "csv", "excel", "sql", "json", "xlsx", "golang", "bash" ]
22-
date-released: 2022-06-30

‎about.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,16 @@ authors:
1111
orcid: "https://orcid.org/0000-0003-0900-6903"
1212

1313
repository-code: "https://github.com/caltechlibrary/datatools"
14-
version: 1.2.3
14+
version: 1.2.4
1515
license-url: "https://data.caltech.edu/license"
1616
keywords: [ "csv", "excel", "sql", "json", "xlsx", "golang", "bash" ]
17-
date-released: 2022-06-30
17+
1818
---
1919

2020
About this software
2121
===================
2222

23-
## datatools 1.2.3
23+
## datatools 1.2.4
2424

2525
### Authors
2626

‎cmd/csv2json/csv2json.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ package main
2222

2323
import (
2424
"encoding/csv"
25-
"encoding/json"
2625
"flag"
2726
"fmt"
2827
"io"
@@ -137,6 +136,7 @@ Convert data1.csv to JSON blobs, one line per blob
137136
trimLeadingSpace bool
138137
fieldsPerRecord int
139138
reuseRecord bool
139+
pretty bool
140140
)
141141

142142
func fmtTxt(src string, appName string, version string) string {
@@ -167,6 +167,7 @@ func main() {
167167
flag.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in fields for CSV input")
168168
flag.BoolVar(&reuseRecord, "reuse-record", false, "reuse the backing array")
169169
flag.IntVar(&fieldsPerRecord, "fields-per-record", 0, "Set the number of fields expected in the CSV read, -1 to turn off")
170+
flag.BoolVar(&pretty, "pretty", false, "pretty print the JSON output")
170171

171172
// Parse environment and options
172173
flag.Parse()
@@ -260,7 +261,12 @@ func main() {
260261
object[fmt.Sprintf("col_%d", col)] = val
261262
}
262263
}
263-
src, err := json.Marshal(object)
264+
var src []byte
265+
if pretty {
266+
src, err = datatools.JSONMarshalIndent(object, "", " ")
267+
} else {
268+
src, err = datatools.JSONMarshal(object)
269+
}
264270
if err != nil {
265271
if !quiet {
266272
fmt.Fprintf(eout, "error row %d, %s\n", rowNo, err)

‎cmd/finddir/finddir.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,6 @@ Find all subdirectories starting with "img".
116116
generateMarkdown bool
117117
generateManPage bool
118118
quiet bool
119-
newLine bool
120-
eol string
121119

122120
// Application Options
123121
showModificationTime bool
@@ -197,8 +195,6 @@ func main() {
197195
flag.StringVar(&outputFName, "o", "", "output filename")
198196
flag.StringVar(&outputFName, "output", "", "output filename")
199197
flag.BoolVar(&quiet, "quiet", false, "suppress error messages")
200-
flag.BoolVar(&newLine, "nl", false, "if true add a trailing newline")
201-
flag.BoolVar(&newLine, "newline", false, "if true add a trailing newline")
202198

203199
// Application Specific Options
204200
flag.BoolVar(&showModificationTime, "m", false, "display file modification time before the path")
@@ -250,9 +246,6 @@ func main() {
250246
fmt.Fprintf(out, "%s %s\n", appName, datatools.Version)
251247
os.Exit(0)
252248
}
253-
if newLine {
254-
eol = "\n"
255-
}
256249

257250
if findPrefix == false && findSuffix == false && findContains == false {
258251
findAll = true
@@ -283,5 +276,4 @@ func main() {
283276
os.Exit(1)
284277
}
285278
}
286-
fmt.Fprintf(out, "%s", eol)
287279
}

‎cmd/findfile/findfile.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,6 @@ Search the current directory and subdirectories for Markdown files with extensio
116116
generateMarkdown bool
117117
generateManPage bool
118118
quiet bool
119-
newLine bool
120-
eol string
121119

122120
// App Specific Options
123121
showModificationTime bool
@@ -197,8 +195,6 @@ func main() {
197195
flag.StringVar(&outputFName, "o", "", "output filename")
198196
flag.StringVar(&outputFName, "output", "", "output filename")
199197
flag.BoolVar(&quiet, "quiet", false, "suppress error messages")
200-
flag.BoolVar(&newLine, "nl", false, "if true add a trailing newline")
201-
flag.BoolVar(&newLine, "newline", false, "if true add a trailing newline")
202198

203199
// App Specific Options
204200
flag.BoolVar(&showModificationTime, "m", false, "display file modification time before the path")
@@ -249,9 +245,6 @@ func main() {
249245
fmt.Fprintf(out, "%s %s\n", appName, datatools.Version)
250246
os.Exit(0)
251247
}
252-
if newLine {
253-
eol = "\n"
254-
}
255248

256249
if findPrefix == false && findSuffix == false && findContains == false {
257250
findAll = true
@@ -282,5 +275,4 @@ func main() {
282275
os.Exit(1)
283276
}
284277
}
285-
fmt.Fprintf(out, "%s", eol)
286278
}

‎cmd/jsoncols/jsoncols.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,9 +282,9 @@ func main() {
282282
row = append(row, result.(json.Number).String())
283283
default:
284284
if prettyPrint {
285-
src, err = json.MarshalIndent(result, "", " ")
285+
src, err = datatools.JSONMarshalIndent(result, "", " ")
286286
} else {
287-
src, err = json.Marshal(result)
287+
src, err = datatools.JSONMarshal(result)
288288
}
289289
if err != nil {
290290
fmt.Fprintln(eout, err)
@@ -338,9 +338,9 @@ func main() {
338338
fmt.Fprintf(out, "%s", result.(json.Number).String())
339339
default:
340340
if prettyPrint {
341-
src, err = json.MarshalIndent(result, "", " ")
341+
src, err = datatools.JSONMarshalIndent(result, "", " ")
342342
} else {
343-
src, err = json.Marshal(result)
343+
src, err = datatools.JSONMarshal(result)
344344
}
345345
if err != nil {
346346
fmt.Fprintln(eout, err)

‎cmd/jsonrange/jsonrange.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
package main
88

99
import (
10-
"encoding/json"
1110
"flag"
1211
"fmt"
1312
"io/ioutil"
@@ -287,7 +286,7 @@ func mapVals(data map[string]interface{}, limit int) ([]string, error) {
287286
if i == limit {
288287
return result, nil
289288
}
290-
outSrc, err := json.Marshal(val)
289+
outSrc, err := datatools.JSONMarshal(val)
291290
if err != nil {
292291
return nil, err
293292
}
@@ -303,7 +302,7 @@ func arrayVals(data []interface{}, limit int) ([]string, error) {
303302
if i == limit {
304303
return result, nil
305304
}
306-
outSrc, err := json.Marshal(val)
305+
outSrc, err := datatools.JSONMarshal(val)
307306
if err != nil {
308307
return nil, err
309308
}

‎cmd/string/string.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ func fnSplit(in io.Reader, out io.Writer, eout io.Writer, args []string) int {
331331
for _, arg := range args {
332332
parts := strings.Split(arg, delimiter)
333333
// Now assemble our JSON array and display it
334-
src, err := json.Marshal(parts)
334+
src, err := datatools.JSONMarshal(parts)
335335
exitOnError(eout, err, quiet)
336336
fmt.Fprintf(out, "%s%s", src, eol)
337337
}
@@ -362,7 +362,7 @@ func fnSplitN(in io.Reader, out io.Writer, eout io.Writer, args []string) int {
362362
parts := strings.SplitN(arg, delimiter, cnt)
363363

364364
// Now assemble our JSON array and display it
365-
src, err := json.Marshal(parts)
365+
src, err := datatools.JSONMarshal(parts)
366366
exitOnError(eout, err, quiet)
367367
fmt.Fprintf(out, "%s%s", src, eol)
368368
}

‎cmd/toml2json/toml2json.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package main
2121

2222
import (
23-
"encoding/json"
2423
"flag"
2524
"fmt"
2625
"io"
@@ -126,9 +125,9 @@ func toml2JSON(in io.Reader, out io.Writer, printPrint bool) error {
126125
return err
127126
}
128127
if prettyPrint == true {
129-
src, err = json.MarshalIndent(m, "", " ")
128+
src, err = datatools.JSONMarshalIndent(m, "", " ")
130129
} else {
131-
src, err = json.Marshal(m)
130+
src, err = datatools.JSONMarshal(m)
132131
}
133132
fmt.Fprintf(out, "%s", src)
134133
return nil

‎cmd/xlsx2json/xlsx2json.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package main
2121

2222
import (
23-
"encoding/json"
2423
"flag"
2524
"fmt"
2625
"io"
@@ -174,7 +173,7 @@ func xlsx2JSON(out io.Writer, workBookName, sheetName string) error {
174173
}
175174
results = append(results, cells)
176175
}
177-
src, err := json.Marshal(results)
176+
src, err := datatools.JSONMarshal(results)
178177
if err != nil {
179178
return err
180179
}

‎cmd/yaml2json/yaml2json.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ func yaml2JSON(in io.Reader, out io.Writer, printPrint bool) error {
139139
if err != nil {
140140
return err
141141
}
142-
src, err = json.MarshalIndent(m, "", " ")
142+
src, err = datatools.JSONMarshalIndent(m, "", " ")
143143
if err != nil {
144144
return err
145145
}

‎codemeta.json

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,11 @@
44
"license": "https://data.caltech.edu/license",
55
"codeRepository": "https://github.com/caltechlibrary/datatools",
66
"dateCreated": "2017-02-06",
7-
"dateModified": "2023-06-30",
8-
"datePublished": "2022-06-30",
7+
"dateRelease": "2023-09-19",
98
"downloadUrl": "https://github.com/caltechlibrary/datatools/releases/",
109
"issueTracker": "https://github.com/caltechlibrary/datatools/issues",
1110
"name": "datatools",
12-
"version": "1.2.3",
11+
"version": "1.2.4",
1312
"description": "A set of command line tools for working with CSV, Excel Workbooks, JSON and structured text documents.",
1413
"applicationCategory": "computer programming",
1514
"developmentStatus": "active",

‎codemeta/codemeta.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package codemeta
22

33
import (
4-
"encoding/json"
54
"fmt"
65
"strings"
76
"time"
@@ -55,11 +54,11 @@ type Codemeta struct {
5554
}
5655

5756
func (person *PersonOrOrganization) ToJSON() ([]byte, error) {
58-
return json.MarshalIndent(person, "", "\t")
57+
return JSONMarshalIndent(person, "", "\t")
5958
}
6059

6160
func (cm *Codemeta) ToJSON() ([]byte, error) {
62-
return json.MarshalIndent(cm, "", "\t")
61+
return JSONMarshalIndent(cm, "", "\t")
6362
}
6463

6564
func (person *PersonOrOrganization) ToCFF() ([]byte, error) {

‎codemeta/json_handlers.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package codemeta
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"io"
7+
)
8+
9+
// JSONUnmarshal is a custom JSON decoder so we can treat numbers easier
10+
func JSONUnmarshal(src []byte, data interface{}) error {
11+
dec := json.NewDecoder(bytes.NewReader(src))
12+
dec.UseNumber()
13+
err := dec.Decode(&data)
14+
if err != nil && err != io.EOF {
15+
return err
16+
}
17+
return nil
18+
}
19+
20+
// JSONMarshal provides provide a custom json encoder to solve a an issue with
21+
// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
22+
func JSONMarshal(data interface{}) ([]byte, error) {
23+
buf := []byte{}
24+
w := bytes.NewBuffer(buf)
25+
enc := json.NewEncoder(w)
26+
enc.SetEscapeHTML(false)
27+
err := enc.Encode(data)
28+
if err != nil {
29+
return nil, err
30+
}
31+
return w.Bytes(), err
32+
}
33+
34+
35+
// JSONMarshalIndent provides provide a custom json encoder to solve a an issue with
36+
// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
37+
func JSONMarshalIndent(data interface{}, prefix string, indent string) ([]byte, error) {
38+
buf := []byte{}
39+
w := bytes.NewBuffer(buf)
40+
enc := json.NewEncoder(w)
41+
enc.SetEscapeHTML(false)
42+
enc.SetIndent(prefix, indent)
43+
err := enc.Encode(data)
44+
if err != nil {
45+
return nil, err
46+
}
47+
return w.Bytes(), err
48+
}
49+
50+

‎datatools.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"bufio"
2424
"bytes"
2525
"encoding/csv"
26-
"encoding/json"
2726
"fmt"
2827
"io"
2928
"strings"
@@ -168,7 +167,7 @@ func Text2Fields(r *bufio.Reader, options *Options) ([]byte, error) {
168167
case AsCSV:
169168
return CSVMarshal(words)
170169
case AsJSON:
171-
return json.Marshal(words)
170+
return JSONMarshal(words)
172171
default:
173172
return []byte(strings.Join(words, options.Delimiter)), nil
174173
}

‎how-to/2col.csv

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
one,three
2+
1,3

‎how-to/3col.csv

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
one,two,three
2+
1,2,3

‎how-to/expected2.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
42
21
"Doe, Jane"
3-
"[email protected]"
2+
"[email protected]"
3+
42

‎how-to/expected3-1.csv

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Number,Value
2+
one,1
3+
two,2
4+
three,3

‎how-to/expected3-3.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
My worksheet 1
2+
My worksheet 2

‎how-to/result3-1.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"Doe, Jane"

‎how-to/result3-2.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"Doe, Jane"|42

‎installer.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# Set the package name and version to install
55
#
66
PACKAGE="datatools"
7-
VERSION="1.2.3"
7+
VERSION="1.2.4"
88
GIT_GROUP="caltechlibrary"
99
RELEASE="https://github.com/$GIT_GROUP/$PACKAGE/releases/tag/v$VERSION"
1010

0 commit comments

Comments
 (0)