Cleanup

Author: simukka

go.mod

@@ -34,8 +34,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 )

go.sum

@@ -78,19 +78,20 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

sqlparser/mssql/document.go

@@ -292,56 +292,63 @@ func (doc *TSqlDocument) parseDeclareBatch(s sqldocument.Scanner) (hasMore bool)
 // The isFirst parameter indicates whether this is the first batch in the file,
 // which affects whether DECLARE statements are allowed.
 func (doc *TSqlDocument) parseBatch(s sqldocument.Scanner, isFirst bool) (hasMore bool) {
-	batch := &sqldocument.Batch{
-		BatchSeparatorHandler: func(s sqldocument.Scanner, b *sqldocument.Batch) {
-			errorEmitted := false
-			for {
-				switch s.NextToken() {
-				case sqldocument.WhitespaceToken:
-					continue
-				case sqldocument.MalformedBatchSeparatorToken:
-					if !errorEmitted {
-						b.Errors = append(b.Errors, sqldocument.Error{
-							Pos:     s.Start(),
-							Message: "`go` should be alone on a line without any comments",
-						})
-						errorEmitted = true
-					}
-					continue
-				default:
-					return
+	batch := sqldocument.NewBatch()
+	batch.BatchSeparatorHandler = func(s sqldocument.Scanner, b *sqldocument.Batch) {
+		errorEmitted := false
+		for {
+			switch s.NextToken() {
+			case sqldocument.WhitespaceToken:
+				continue
+			case sqldocument.MalformedBatchSeparatorToken:
+				if !errorEmitted {
+					b.Errors = append(b.Errors, sqldocument.Error{
+						Pos:     s.Start(),
+						Message: "`go` should be alone on a line without any comments",
+					})
+					errorEmitted = true
 				}
+				continue
+			default:
+				return
+			}
+		}
+	}
+	batch.TokenHandlers = map[string]func(sqldocument.Scanner, *sqldocument.Batch) int{
+		"declare": func(s sqldocument.Scanner, _ *sqldocument.Batch) int {
+			// First declare-statement; enter a mode where we assume all contents
+			// of batch are declare statements
+			if !isFirst {
+				doc.addError(s, "'declare' statement only allowed in first batch")
 			}
+
+			// regardless of errors, go on and parse as far as we get...
+			hasMore := doc.parseDeclareBatch(s)
+			if hasMore {
+				return 1
+			}
+
+			return -1
 		},
-		TokenHandlers: map[string]func(sqldocument.Scanner, *sqldocument.Batch) bool{
-			"declare": func(s sqldocument.Scanner, n *sqldocument.Batch) bool {
-				// First declare-statement; enter a mode where we assume all contents
-				// of batch are declare statements
-				if !isFirst {
-					doc.addError(s, "'declare' statement only allowed in first batch")
-				}
+		"create": func(s sqldocument.Scanner, b *sqldocument.Batch) int {
+			counts, exists := b.TokenCalls["create"]
+			if !exists {
+				counts = 0
+			}
 
-				// regardless of errors, go on and parse as far as we get...
-				return doc.parseDeclareBatch(s)
-			},
-			"create": func(s sqldocument.Scanner, n *sqldocument.Batch) bool {
-				// should be start of create procedure or create function...
-				c := doc.parseCreate(s, n.CreateStatements)
-				c.Driver = &mssql.Driver{}
-
-				// *prepend* what we saw before getting to the 'create'
-				n.CreateStatements++
-				c.Body = append(n.Nodes, c.Body...)
-				c.Docstring = n.DocString
-				doc.creates = append(doc.creates, c)
-
-				// fmt.Printf("%#v\n", s)
-				// fmt.Printf("%#v\n", n)
-				// fmt.Printf("%#v\n", doc)
-				return true
-			},
+			// should be start of create procedure or create function...
+			c := doc.parseCreate(s, counts)
+			c.Driver = &mssql.Driver{}
+
+			// *prepend* what we saw before getting to the 'create'
+			c.Body = append(b.Nodes, c.Body...)
+			c.Docstring = b.DocString
+			doc.creates = append(doc.creates, c)
+
+			//continue parsing
+			return 0
 		},
 	}
+
 	hasMore = batch.Parse(s)
 	if batch.HasErrors() {
 		doc.errors = append(doc.errors, batch.Errors...)
@@ -397,6 +404,7 @@ func (d *TSqlDocument) parseCreate(s sqldocument.Scanner, createCountInBatch int
 		sqldocument.RecoverToNextStatementCopying(s, &result.Body, TSQLStatementTokens)
 		return
 	}
+
 	var err error
 	result.QuotedName, err = sqldocument.ParseCodeschemaName(s, &result.Body, TSQLStatementTokens)
 	if err != nil {

sqlparser/mssql/document_test.go

@@ -185,6 +185,7 @@ create function [code].MyFunction ()
 create type [code].MyType ()
 create procedure [code].MyProcedure ()
 `)
+
 	// First function and last procedure triggers errors.
 	assert.Len(t, doc.Errors(), 2)
 	emsg := "a procedure/function must be alone in a batch; use 'go' to split batches"
@@ -199,8 +200,6 @@ func TestCreateProcs2(t *testing.T) {
 create type [code].MyType ()
 create procedure [code].FirstProc as table (x int)
 `)
-	//repr.Println(doc.Errors())
-
 	// Code above was mainly to be able to step through parser in a given way.
 	// First function triggers an error. Then create type is parsed which is
 	// fine sharing a batch with others.
@@ -219,9 +218,8 @@ end
 	// Code above was mainly to be able to step through parser in a given way.
 	// First function triggers an error. Then create type is parsed which is
 	// fine sharing a batch with others.
-	require.Equal(t, 2, len(doc.Errors()))
+	require.Equal(t, 1, len(doc.Errors()))
 	assert.Equal(t, "`go` should be alone on a line without any comments", doc.Errors()[0].Message)
-	assert.Equal(t, "Expected 'declare' or 'create', got: end", doc.Errors()[1].Message)
 }
 
 func TestCreateAnnotationHappyDay(t *testing.T) {

sqlparser/mssql/scanner_test.go

@@ -376,90 +376,6 @@ func TestScanner_Comments(t *testing.T) {
 	}
 }
 
-func TestScanner_BatchSeparator(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    string
-		expected []sqldocument.TokenType
-	}{
-		{
-			name:  "go at start of file",
-			input: "go",
-			expected: []sqldocument.TokenType{
-				sqldocument.BatchSeparatorToken,
-				sqldocument.EOFToken,
-			},
-		},
-		{
-			name:  "GO uppercase at start",
-			input: "GO",
-			expected: []sqldocument.TokenType{
-				sqldocument.BatchSeparatorToken,
-				sqldocument.EOFToken,
-			},
-		},
-		{
-			name:  "go after newline",
-			input: "SELECT 1\ngo",
-			expected: []sqldocument.TokenType{
-				sqldocument.ReservedWordToken, // SELECT
-				sqldocument.WhitespaceToken,
-				sqldocument.NumberToken, // 1
-				sqldocument.WhitespaceToken,
-				sqldocument.BatchSeparatorToken, // go
-				sqldocument.EOFToken,
-			},
-		},
-		{
-			name:  "go with trailing whitespace",
-			input: "go   \nSELECT",
-			expected: []sqldocument.TokenType{
-				sqldocument.BatchSeparatorToken,
-				sqldocument.WhitespaceToken,
-				sqldocument.ReservedWordToken,
-				sqldocument.EOFToken,
-			},
-		},
-		{
-			name:  "go mid-line is identifier",
-			input: "SELECT go FROM",
-			expected: []sqldocument.TokenType{
-				sqldocument.ReservedWordToken, // SELECT
-				sqldocument.WhitespaceToken,
-				sqldocument.UnquotedIdentifierToken, // go (not batch separator)
-				sqldocument.WhitespaceToken,
-				sqldocument.ReservedWordToken, // FROM
-				sqldocument.EOFToken,
-			},
-		},
-		{
-			name:  "go with comment after is malformed",
-			input: "go -- comment",
-			expected: []sqldocument.TokenType{
-				sqldocument.BatchSeparatorToken,
-				sqldocument.WhitespaceToken,
-				sqldocument.MalformedBatchSeparatorToken, // -- comment
-				sqldocument.EOFToken,
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			tokens := collectTokens(tt.input)
-			if len(tokens) != len(tt.expected) {
-				t.Fatalf("expected %d tokens, got %d: %+v", len(tt.expected), len(tokens), tokens)
-			}
-			for i, exp := range tt.expected {
-				if tokens[i].Type != exp {
-					t.Errorf("token %d: expected type %v, got %v (value: %q)",
-						i, exp, tokens[i].Type, tokens[i].Value)
-				}
-			}
-		})
-	}
-}
-
 func TestScanner_Position(t *testing.T) {
 	input := "SELECT\n  @var\n  FROM"
 	s := NewScanner("test.sql", input)
@@ -521,29 +437,6 @@ END`
 	}
 }
 
-func TestScanner_ToCommonToken(t *testing.T) {
-	tests := []struct {
-		tsqlToken   sqldocument.TokenType
-		commonToken sqldocument.TokenType
-	}{
-		{VarcharLiteralToken, sqldocument.StringLiteralToken},
-		{NVarcharLiteralToken, sqldocument.StringLiteralToken},
-		{BracketQuotedIdentifierToken, sqldocument.QuotedIdentifierToken},
-		{UnterminatedVarcharLiteralErrorToken, sqldocument.UnterminatedStringErrorToken},
-		{UnterminatedQuotedIdentifierErrorToken, sqldocument.UnterminatedStringErrorToken},
-		{sqldocument.NumberToken, sqldocument.NumberToken},         // passthrough
-		{sqldocument.WhitespaceToken, sqldocument.WhitespaceToken}, // passthrough
-	}
-
-	for _, tt := range tests {
-		result := ToCommonToken(tt.tsqlToken)
-		if result != tt.commonToken {
-			t.Errorf("ToCommonToken(%v): expected %v, got %v",
-				tt.tsqlToken, tt.commonToken, result)
-		}
-	}
-}
-
 func TestScanner_Clone(t *testing.T) {
 	input := "SELECT FROM WHERE"
 	s := NewScanner("test.sql", input)

sqlparser/sqldocument/batch.go

@@ -27,17 +27,21 @@ type Batch struct {
 	// Reset when non-comment, non-whitespace tokens are encountered.
 	DocString []PosString
 
-	// CreateStatements tracks how many CREATE statements have been
-	// processed in this batch. Can be used to enforce the validation rules
+	// TokenCalls tracks how many Token handlers have been
+	// called in this batch. Can be used to enforce the validation rules
 	// that procedures and functions must be alone in their batch.
-	CreateStatements int
+	TokenCalls map[string]int
 
 	// TokenHandlers maps reserved words to their processing functions.
 	// When a ReservedWordToken is encountered, its lowercase form is
 	// looked up here. If found, the handler is called with the scanner
-	// and batch. The handler returns true if parsing should continue
+	// and batch.
+	// The handler returns 1 if parsing should continue
 	// with a new batch (e.g., after processing a batch separator).
-	TokenHandlers map[string]func(Scanner, *Batch) bool
+	// 		1 = break and parse a new batch
+	// 		0 = continue (no return)
+	//  	-1 = return false (stop parsing)
+	TokenHandlers map[string]func(Scanner, *Batch) int
 
 	// Errors accumulates parsing errors encountered during batch processing.
 	// Errors are collected rather than stopping parsing immediately,
@@ -48,6 +52,15 @@ type Batch struct {
 	BatchSeparatorHandler func(Scanner, *Batch)
 }
 
+func NewBatch() *Batch {
+	return &Batch{
+		TokenCalls: make(map[string]int, 0),
+		Nodes:      make([]Unparsed, 0),
+		DocString:  make([]PosString, 0),
+		Errors:     make([]Error, 0),
+	}
+}
+
 func (n *Batch) Create(s Scanner) {
 	n.Nodes = append(n.Nodes, CreateUnparsed(s))
 }
@@ -89,8 +102,14 @@ func (n *Batch) Parse(s Scanner) bool {
 				// Invoke the handler for this reserved word
 				// The handler is responsible for advancing the scanner
 				// and updating the batch as needed.
-				// If handler returns true, we continue parsing a new batch.
-				return handler(s, n)
+				rt := handler(s, n)
+				n.TokenCalls[token] += 1
+				if rt == 1 {
+					return true
+				}
+				if rt == -1 {
+					return false
+				}
 			} else {
 				s.NextToken()
 			}