Handle Unicode escapes up to 2^31 - 1

internal/lualex/lex.go

@@ -388,7 +388,7 @@ func (s *Scanner) shortLiteralString(end byte) (string, error) {
 				}
 				r = r<<4 | rune(nibble)
 			}
-			sb.WriteRune(r)
+			writeRune(sb, r)
 		default:
 			if !isDigit(b) {
 				return sb.String(), fmt.Errorf("%v: invalid escape sequence", s.prev)
@@ -807,6 +807,30 @@ func hexDigit(c byte) (byte, error) {
 	}
 }
 
+// writeRune encodes a rune as UTF-8,
+// permitting runes up to 1<<31 - 1.
+func writeRune(sb *strings.Builder, c rune) {
+	if 0 <= c && c < 0x80 {
+		sb.WriteByte(byte(c))
+		return
+	}
+
+	var buf [6]byte
+	firstByteMax := byte(0x3f)
+	n := 1
+	for {
+		buf[len(buf)-n] = byte(0x80 | (c & 0x3f))
+		n++
+		c >>= 6
+		firstByteMax >>= 1
+		if c <= rune(firstByteMax) {
+			break
+		}
+	}
+	buf[len(buf)-n] = (^firstByteMax << 1) | byte(c)
+	sb.Write(buf[len(buf)-n:])
+}
+
 type longLiteralWriter struct {
 	sb   strings.Builder
 	prev byte

internal/lualex/lex_test.go

@@ -337,6 +337,56 @@ func TestScanner(t *testing.T) {
 	}
 }
 
+func TestUnquote(t *testing.T) {
+	tests := []struct {
+		s    string
+		want string
+		err  bool
+	}{
+		{
+			s:    `""`,
+			want: "",
+		},
+		{
+			s:    `''`,
+			want: "",
+		},
+		{
+			s:    `"abc"`,
+			want: "abc",
+		},
+		{
+			s:    `'abc'`,
+			want: "abc",
+		},
+
+		// Invalid UTF-8 code points.
+		{
+			s:    `"\u{110000}"`,
+			want: "\xf4\x90\x80\x80",
+		},
+		{
+			s:    `"\u{7FFFFFFF}"`,
+			want: "\xfd\xbf\xbf\xbf\xbf\xbf",
+		},
+		{
+			s:   `"\u{80000000}"`,
+			err: true,
+		},
+	}
+
+	for _, test := range tests {
+		got, err := Unquote(test.s)
+		if got != test.want || (err != nil) != test.err {
+			errString := "<nil>"
+			if test.err {
+				errString = "<error>"
+			}
+			t.Errorf("Unquote(%q) = %q, %v; want %q, %s", test.s, got, err, test.want, errString)
+		}
+	}
+}
+
 func FuzzQuote(f *testing.F) {
 	f.Add("")
 	f.Add("abc")