move lexer to package

- Keep it 100
rread · Feb 10, 2016 · 23becfd · 23becfd
1 parent e78aebf
commit 23becfd
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 18 deletions.
diff --git a/eval.go b/eval.go
@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/rread/rsi/lexer"
 	"github.com/rread/rsi/log"
 )
 
@@ -65,7 +66,7 @@ func nullp(d Data) Boolean {
 }
 
 type Tokenizer interface {
-	NextItem() *TokenItem
+	NextItem() *lexer.TokenItem
 }
 
 var (
@@ -82,33 +83,33 @@ func read(l Tokenizer) (Data, error) {
 	}
 	//log.Debugf("scan: %v\n", t)
 	switch t.Token {
-	case LEFT_PAREN:
+	case lexer.LEFT_PAREN:
 		return readList(l)
-	case RIGHT_PAREN:
+	case lexer.RIGHT_PAREN:
 		return nil, nil
-	case SYMBOL:
+	case lexer.SYMBOL:
 		return internSymbol(t.Lit), nil
-	case QUOTE:
+	case lexer.QUOTE:
 		return readQuote(l)
-	case NUMBER:
+	case lexer.NUMBER:
 		v, err := strconv.ParseFloat(t.Lit, 64)
 		if err != nil {
 			log.Fatal("Number fail:", err)
 		}
 		return Number(v), nil
-	case EOF:
+	case lexer.EOF:
 		return nil, ErrorEOF
-	case STRING:
+	case lexer.STRING:
 		return StringWithValue(t.Lit), nil
-	case TRUE:
+	case lexer.TRUE:
 		return T, nil
-	case FALSE:
+	case lexer.FALSE:
 		return False, nil
-	case DOT:
+	case lexer.DOT:
 		return _dot, nil
-	case ILLEGAL:
+	case lexer.ILLEGAL:
 		return nil, errors.New(t.Lit)
-	case COMMENT:
+	case lexer.COMMENT:
 		return read(l)
 	}
 	return nil, errors.New("Malformed input")
@@ -474,7 +475,7 @@ func replReader(in io.Reader, env *Env) (Data, error) {
 	//	l := NewScanner(in)
 	buf := make([]byte, 1024)
 	n, _ := in.Read(buf)
-	l := NewLexer("lispy", string(buf[:n]))
+	l := lexer.New("lispy", string(buf[:n]))
 	var result Data
 	for {
 		var err error

diff --git a/lexer_test.go → lexer/lexer_test.go b/lexer_test.go → lexer/lexer_test.go
@@ -1,4 +1,4 @@
-package main
+package lexer
 
 import (
 	"testing"
@@ -19,15 +19,24 @@ func TestLexer(t *testing.T) {
 			{"", `EOF ""`, ""},
 			{"\n  ", `EOF ""`, ""},
 			{"; asdf", `COMMENT "; asdf"`, ""},
+			{"; asdf\n", `COMMENT "; asdf\n"`, ""},
 			{"(", `LEFT_PAREN "("`, ""},
 			{")", `RIGHT_PAREN ")"`, ""},
 			{" . ", `DOT "."`, ""},
 			{"'", `QUOTE "'"`, ""},
+			{"-", `SYMBOL "-"`, ""},
+			{"-1", `NUMBER "-1"`, ""},
+			{".", `NUMBER "."`, ""},
 			{"123", `NUMBER "123"`, ""},
+			{"-abc", `SYMBOL "-abc"`, ""},
 			{"abc", `SYMBOL "abc"`, ""},
 			{`"abc"`, `STRING "abc"`, ""},
+			{`"abc\`, `ILLEGAL "unterminated string: \"abc\""`, ""},
 			{"#t", `TRUE "t"`, ""},
 			{"#f", `FALSE "f"`, ""},
+			{"#n", `ILLEGAL "unsupported hash code #n"`, ""},
+			{"a(", `SYMBOL "a"`, ""},
+			{"12(", `NUMBER "12"`, ""},
 		} {
 			doLex(c.value, c.expected, c.err)
 		}
@@ -41,7 +50,7 @@ func TestLexer(t *testing.T) {
 }
 
 func doLex(expr, expectedValue, expectedError string) {
-	l := NewLexer("test", expr)
+	l := New("test", expr)
 	ti := l.NextItem()
 	So(ti.String(), ShouldEqual, expectedValue)
 

diff --git a/scanner.go → lexer/scanner.go b/scanner.go → lexer/scanner.go
@@ -1,4 +1,4 @@
-package main
+package lexer
 
 import (
 	"fmt"
@@ -94,7 +94,7 @@ type Lexer struct {
 	items chan *TokenItem
 }
 
-func NewLexer(name, input string) *Lexer {
+func New(name, input string) *Lexer {
 	l := &Lexer{
 		name:  name,
 		input: input,