Can't use ${field_name} if it contains UTF-8 characters also encodeab…

…le as Latin-1 (#1363) * unit-test data * docgen * windows unit-test accommodations
johnkerl · Aug 20, 2023 · 2107d52 · 2107d52
1 parent 9d1d2e0
commit 2107d52
Show file tree

Hide file tree

Showing 24 changed files with 46 additions and 8 deletions.
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
@@ -3470,5 +3470,5 @@ MILLER(1)                                                            MILLER(1)
 
 
 
-                                  2023-08-19                         MILLER(1)
+                                  2023-08-20                         MILLER(1)
 </pre>
diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
@@ -3449,4 +3449,4 @@ MILLER(1)                                                            MILLER(1)
 
 
 
-                                  2023-08-19                         MILLER(1)
+                                  2023-08-20                         MILLER(1)
diff --git a/internal/pkg/parsing/lexer/lexer.go b/internal/pkg/parsing/lexer/lexer.go
diff --git a/internal/pkg/parsing/lexer/transitiontable.go b/internal/pkg/parsing/lexer/transitiontable.go
diff --git a/internal/pkg/parsing/mlr.bnf b/internal/pkg/parsing/mlr.bnf
@@ -271,6 +271,7 @@ _braced_char
   | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '['
   | ']' | '^' | '_' | '`'       | '|'       | '~'
   | ( '\\' '{' ) | ( '\\' '}' )
+  | '\u00a0'-'\u00ff'
   | '\u0100'-'\U0010FFFF'
 ;
 braced_field_name: '$' '{' _braced_char { _braced_char } '}' ;

diff --git a/man/manpage.txt b/man/manpage.txt
@@ -3449,4 +3449,4 @@ MILLER(1)                                                            MILLER(1)
 
 
 
-                                  2023-08-19                         MILLER(1)
+                                  2023-08-20                         MILLER(1)
diff --git a/man/mlr.1 b/man/mlr.1
@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2023-08-19
+.\"      Date: 2023-08-20
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2023-08-19" "\ \&" "\ \&"
+.TH "MILLER" "1" "2023-08-20" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/test/cases/dsl-utf8-field-names/0001/cmd b/test/cases/dsl-utf8-field-names/0001/cmd
@@ -0,0 +1 @@
+mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv
diff --git a/test/cases/dsl-utf8-field-names/0001/experr b/test/cases/dsl-utf8-field-names/0001/experr
diff --git a/test/cases/dsl-utf8-field-names/0001/expout b/test/cases/dsl-utf8-field-names/0001/expout
@@ -0,0 +1,3 @@
+año  ποσότητα
+2021 130
+2022 145
diff --git a/test/cases/dsl-utf8-field-names/0001/mlr b/test/cases/dsl-utf8-field-names/0001/mlr
@@ -0,0 +1 @@
+$año > 2020
diff --git a/test/cases/dsl-utf8-field-names/0002/cmd b/test/cases/dsl-utf8-field-names/0002/cmd
@@ -0,0 +1 @@
+mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv
diff --git a/test/cases/dsl-utf8-field-names/0002/experr b/test/cases/dsl-utf8-field-names/0002/experr
diff --git a/test/cases/dsl-utf8-field-names/0002/expout b/test/cases/dsl-utf8-field-names/0002/expout
@@ -0,0 +1,3 @@
+año  ποσότητα
+2021 130
+2022 145
diff --git a/test/cases/dsl-utf8-field-names/0002/mlr b/test/cases/dsl-utf8-field-names/0002/mlr
@@ -0,0 +1 @@
+${año} > 2020
diff --git a/test/cases/dsl-utf8-field-names/0003/cmd b/test/cases/dsl-utf8-field-names/0003/cmd
@@ -0,0 +1 @@
+mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv
diff --git a/test/cases/dsl-utf8-field-names/0003/experr b/test/cases/dsl-utf8-field-names/0003/experr
diff --git a/test/cases/dsl-utf8-field-names/0003/expout b/test/cases/dsl-utf8-field-names/0003/expout
@@ -0,0 +1,3 @@
+año  ποσότητα
+2021 130
+2022 145
diff --git a/test/cases/dsl-utf8-field-names/0003/mlr b/test/cases/dsl-utf8-field-names/0003/mlr
@@ -0,0 +1 @@
+$ποσότητα > 100
diff --git a/test/cases/dsl-utf8-field-names/0004/cmd b/test/cases/dsl-utf8-field-names/0004/cmd
@@ -0,0 +1 @@
+mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv
diff --git a/test/cases/dsl-utf8-field-names/0004/experr b/test/cases/dsl-utf8-field-names/0004/experr
diff --git a/test/cases/dsl-utf8-field-names/0004/expout b/test/cases/dsl-utf8-field-names/0004/expout
@@ -0,0 +1,3 @@
+año  ποσότητα
+2021 130
+2022 145
diff --git a/test/cases/dsl-utf8-field-names/0004/mlr b/test/cases/dsl-utf8-field-names/0004/mlr
@@ -0,0 +1 @@
+${ποσότητα} > 100
diff --git a/test/input/datos-plurilingües.csv b/test/input/datos-plurilingües.csv
@@ -0,0 +1,4 @@
+año,ποσότητα
+2020,100
+2021,130
+2022,145