Add comment support cloudspannerecosystem#15

pkg/spanner/client_test.go

@@ -63,23 +63,41 @@ const (
 )
 
 func TestLoadDDL(t *testing.T) {
-	ctx := context.Background()
+	tests := map[string]struct {
+		schema string
+		want   string
+	}{
+		"If schema has no comment strings, it should work without any errors": {
+			"testdata/schema.sql",
+			"testdata/schema.sql",
+		},
+		"If schema has comment strings, it should work without any errors": {
+			"testdata/schema_with_comment.sql",
+			"testdata/schema.sql",
+		},
+	}
+	for name, tt := range tests {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			ctx := context.Background()
 
-	client, done := testClientWithDatabase(t, ctx)
-	defer done()
+			client, done := testClientWithDatabaseByFileName(t, ctx, tt.schema)
+			defer done()
 
-	gotDDL, err := client.LoadDDL(ctx)
-	if err != nil {
-		t.Fatalf("failed to load ddl: %v", err)
-	}
+			gotDDL, err := client.LoadDDL(ctx)
+			if err != nil {
+				t.Fatalf("failed to load ddl: %v", err)
+			}
 
-	wantDDL, err := ioutil.ReadFile("testdata/schema.sql")
-	if err != nil {
-		t.Fatalf("failed to read ddl file: %v", err)
-	}
+			wantDDL, err := ioutil.ReadFile(tt.want)
+			if err != nil {
+				t.Fatalf("failed to read ddl file: %v", err)
+			}
 
-	if want, got := string(wantDDL), string(gotDDL); want != got {
-		t.Errorf("want: \n%s\n but got: \n%s", want, got)
+			if want, got := string(wantDDL), string(gotDDL); want != got {
+				t.Errorf("want: \n%s\n but got: \n%s", want, got)
+			}
+		})
 	}
 }
 
@@ -387,7 +405,7 @@ func TestEnsureMigrationTable(t *testing.T) {
 	}
 }
 
-func testClientWithDatabase(t *testing.T, ctx context.Context) (*Client, func()) {
+func testClientWithDatabaseByFileName(t *testing.T, ctx context.Context, schema string) (*Client, func()) {
 	t.Helper()
 
 	project := os.Getenv(envSpannerProjectID)
@@ -417,7 +435,7 @@ func testClientWithDatabase(t *testing.T, ctx context.Context) (*Client, func())
 		t.Fatalf("failed to create spanner client: %v", err)
 	}
 
-	ddl, err := ioutil.ReadFile("testdata/schema.sql")
+	ddl, err := ioutil.ReadFile(schema)
 	if err != nil {
 		t.Fatalf("failed to read schema file: %v", err)
 	}
@@ -434,3 +452,7 @@ func testClientWithDatabase(t *testing.T, ctx context.Context) (*Client, func())
 		}
 	}
 }
+
+func testClientWithDatabase(t *testing.T, ctx context.Context) (*Client, func()) {
+	return testClientWithDatabaseByFileName(t, ctx, "testdata/schema.sql")
+}

pkg/spanner/migration.go

@@ -20,7 +20,6 @@
 package spanner
 
 import (
-	"bytes"
 	"errors"
 	"io/ioutil"
 	"path/filepath"
@@ -43,6 +42,10 @@ var (
 	MigrationNameRegex = regexp.MustCompile(`[a-zA-Z0-9_\-]+`)
 
 	dmlRegex = regexp.MustCompile("^(UPDATE|DELETE)[\t\n\f\r ].*")
+
+	commentsRemovalRegex = regexp.MustCompile("(#[^\r\n]*|\\-\\-[^\r\n]*|\\/\\*[^\\*/]*?\\*\\/)")
+
+	commentsWhiteSpaceRemovalRegex = regexp.MustCompile("(?m)^\\\\s*\\r?\\n")
 )
 
 const (
@@ -127,18 +130,23 @@ func LoadMigrations(dir string) (Migrations, error) {
 }
 
 func toStatements(file []byte) []string {
-	contents := bytes.Split(file, []byte(statementsSeparator))
+	commentRemmoved := removeComments(string(file))
+	contents := strings.Split(commentRemmoved, statementsSeparator)
 
 	statements := make([]string, 0, len(contents))
 	for _, c := range contents {
 		if statement := strings.TrimSpace(string(c)); statement != "" {
 			statements = append(statements, statement)
 		}
 	}
-
 	return statements
 }
 
+func removeComments(statement string) string {
+	s := commentsRemovalRegex.ReplaceAllString(statement, "")
+	return commentsWhiteSpaceRemovalRegex.ReplaceAllString(s, "")
+}
+
 func inspectStatementsKind(statements []string) (statementKind, error) {
 	kindMap := map[statementKind]uint64{
 		statementKindDDL: 0,

pkg/spanner/testdata/schema.sql

@@ -7,3 +7,7 @@ CREATE TABLE Singers (
   SingerID STRING(36) NOT NULL,
   FirstName STRING(1024),
 ) PRIMARY KEY(SingerID);
+
+CREATE TABLE TableWithComments (
+  ID STRING(36) NOT NULL,
+) PRIMARY KEY(ID);

pkg/spanner/testdata/schema_with_comment.sql

@@ -0,0 +1,21 @@
+CREATE TABLE SchemaMigrations (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+) PRIMARY KEY(Version);
+
+CREATE TABLE Singers (
+  SingerID STRING(36) NOT NULL,
+  FirstName STRING(1024),
+) PRIMARY KEY(SingerID);
+
+# this is an inline comment
+CREATE TABLE TableWithComments (
+  /* this is a multiline comment
+  on two lines */
+  ID STRING(36) NOT NULL, -- this is an inline comment
+  /* this is an inline comment */
+
+  /* column commented out
+  Name STRING(36) NOT NULL
+   */
+) PRIMARY KEY(ID); # another comment