Skip to content

Commit

Permalink
Added unit tests (#566)
Browse files Browse the repository at this point in the history 
* added unit tests

* added unit tests
  • Loading branch information
@shreyakhajanchi
shreyakhajanchi authored Jun 28, 2023
1 parent e4e33b1 commit 142af83
Show file tree
Hide file tree
Showing 11 changed files with 301 additions and 12 deletions.
64 changes: 64 additions & 0 deletions sources/mysql/infoschema_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,70 @@ func TestProcessData_MultiCol(t *testing.T) {
assert.Equal(t, int64(0), conv.Unexpecteds())
}

func TestProcessSchema_Sharded(t *testing.T) {
// Tests multi-column behavior of ProcessSQLData (including
// handling of null columns and synthetic keys). Also tests
// the combination of ProcessInfoSchema and ProcessSQLData
// i.e. ProcessSQLData uses the schemas built by
// ProcessInfoSchema.
ms := []mockSpec{
{
query: "SELECT table_name FROM information_schema.tables where table_type = 'BASE TABLE' and (.+)",
args: []driver.Value{"test"},
cols: []string{"table_name"},
rows: [][]driver.Value{{"test"}},
}, {
query: "SELECT (.+) FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS (.+)",
args: []driver.Value{"test", "test"},
cols: []string{"column_name", "constraint_type"},
rows: [][]driver.Value{}, // No primary key --> force generation of synthetic key.
}, {
query: "SELECT (.+) FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS (.+)",
args: []driver.Value{"test", "test"},
cols: []string{"REFERENCED_TABLE_NAME", "COLUMN_NAME", "REFERENCED_COLUMN_NAME", "CONSTRAINT_NAME"},
}, {
query: "SELECT (.+) FROM information_schema.COLUMNS (.+)",
args: []driver.Value{"test", "test"},
cols: []string{"column_name", "data_type", "column_type", "is_nullable", "column_default", "character_maximum_length", "numeric_precision", "numeric_scale", "extra"},
rows: [][]driver.Value{
{"a", "text", "text", "NO", nil, nil, nil, nil, nil},
{"b", "double", "double", "YES", nil, nil, 53, nil, nil},
{"c", "bigint", "bigint", "YES", nil, nil, 64, 0, nil}},
},
{
query: "SELECT (.+) FROM INFORMATION_SCHEMA.STATISTICS (.+)",
args: []driver.Value{"test", "test"},
cols: []string{"INDEX_NAME", "COLUMN_NAME", "SEQ_IN_INDEX", "COLLATION", "NON_UNIQUE"},
},
{
query: "SELECT (.+) FROM `test`.`test`",
cols: []string{"a", "b", "c"},
rows: [][]driver.Value{
{"cat", 42.3, nil},
{"dog", nil, 22}},
},
}
db := mkMockDB(t, ms)
conv := internal.MakeConv()
isi := InfoSchemaImpl{"test", db, profiles.SourceProfile{}, profiles.TargetProfile{}}
err := common.ProcessSchema(conv, isi, 1, internal.AdditionalSchemaAttributes{IsSharded: true})
assert.Nil(t, err)
expectedSchema := map[string]ddl.CreateTable{
"test": {
Name: "test",
ColIds: []string{"a", "b", "c", "synth_id"},
ColDefs: map[string]ddl.ColumnDef{
"a": {Name: "a", T: ddl.Type{Name: ddl.String, Len: ddl.MaxLength}, NotNull: true},
"b": {Name: "b", T: ddl.Type{Name: ddl.Float64}},
"c": {Name: "c", T: ddl.Type{Name: ddl.Int64}},
"synth_id": {Name: "synth_id", T: ddl.Type{Name: ddl.String, Len: 50}},
"migration_shard_id": {Name: "migration_shard_id", T: ddl.Type{Name: ddl.String, Len: 50}},
},
PrimaryKeys: []ddl.IndexKey{{ColId: "synth_id", Order: 1}}},
}
internal.AssertSpSchema(conv, t, expectedSchema, stripSchemaComments(conv.SpSchema))
}

func TestSetRowStats(t *testing.T) {
ms := []mockSpec{
{
Expand Down
15 changes: 8 additions & 7 deletions streaming/streaming.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,13 +500,14 @@ func createLaunchParameters(dataflowCfg DataflowCfg, inputFilePattern string, pr
JobName: dataflowCfg.JobName,
Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://dataflow-templates-southamerica-west1/2023-03-07-00_RC00/flex/Cloud_Datastream_to_Spanner"},
Parameters: map[string]string{
"inputFilePattern": inputFilePattern,
"streamName": fmt.Sprintf("projects/%s/locations/%s/streams/%s", project, datastreamCfg.StreamLocation, datastreamCfg.StreamId),
"instanceId": instance,
"databaseId": dbName,
"sessionFilePath": streamingCfg.TmpDir + "session.json",
"deadLetterQueueDirectory": inputFilePattern + "dlq",
"transformationContextFilePath": streamingCfg.TmpDir + "transformationContext.json",
"inputFilePattern": inputFilePattern,
"streamName": fmt.Sprintf("projects/%s/locations/%s/streams/%s", project, datastreamCfg.StreamLocation, datastreamCfg.StreamId),
"instanceId": instance,
"databaseId": dbName,
"sessionFilePath": streamingCfg.TmpDir + "session.json",
"deadLetterQueueDirectory": inputFilePattern + "dlq",
// TODO(khajanchi): Uncomment this one dataflow template is released
// "transformationContextFilePath": streamingCfg.TmpDir + "transformationContext.json",
},
Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
MaxWorkers: maxWorkers,
Expand Down
4 changes: 2 additions & 2 deletions ui/dist/ui/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
<link rel="preconnect" href="https://fonts.gstatic.com">
<style type="text/css">@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fCRc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0460-052F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fABc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fCBc4AMP6lbBP.woff2) format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fBxc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0370-03FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fCxc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fChc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:300;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmSU5fBBc4AMP6lQ.woff2) format('woff2');unicode-range:U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu72xKKTU1Kvnz.woff2) format('woff2');unicode-range:U+0460-052F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu5mxKKTU1Kvnz.woff2) format('woff2');unicode-range:U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu7mxKKTU1Kvnz.woff2) format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4WxKKTU1Kvnz.woff2) format('woff2');unicode-range:U+0370-03FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu7WxKKTU1Kvnz.woff2) format('woff2');unicode-range:U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu7GxKKTU1Kvnz.woff2) format('woff2');unicode-range:U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:400;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxKKTU1Kg.woff2) format('woff2');unicode-range:U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fCRc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0460-052F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fABc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fCBc4AMP6lbBP.woff2) format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fBxc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0370-03FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fCxc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fChc4AMP6lbBP.woff2) format('woff2');unicode-range:U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF;}@font-face{font-family:'Roboto';font-style:normal;font-weight:500;font-display:swap;src:url(https://fonts.gstatic.com/s/roboto/v30/KFOlCnqEu92Fr1MmEU9fBBc4AMP6lQ.woff2) format('woff2');unicode-range:U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;}</style>
<style type="text/css">@font-face{font-family:'Material Icons';font-style:normal;font-weight:400;src:url(https://fonts.gstatic.com/s/materialicons/v140/flUhRq6tzZclQEJ-Vdg-IuiaDsNcIhQ8tQ.woff2) format('woff2');}@font-face{font-family:'Material Icons Outlined';font-style:normal;font-weight:400;src:url(https://fonts.gstatic.com/s/materialiconsoutlined/v109/gok-H7zzDkdnRel8-DQ6KAXJ69wP1tGnf4ZGhUcel5euIg.woff2) format('woff2');}.material-icons{font-family:'Material Icons';font-weight:normal;font-style:normal;font-size:24px;line-height:1;letter-spacing:normal;text-transform:none;display:inline-block;white-space:nowrap;word-wrap:normal;direction:ltr;-webkit-font-feature-settings:'liga';-webkit-font-smoothing:antialiased;}.material-icons-outlined{font-family:'Material Icons Outlined';font-weight:normal;font-style:normal;font-size:24px;line-height:1;letter-spacing:normal;text-transform:none;display:inline-block;white-space:nowrap;word-wrap:normal;direction:ltr;-webkit-font-feature-settings:'liga';-webkit-font-smoothing:antialiased;}</style>
<style>.mat-typography{font:400 14px/20px Roboto,Helvetica Neue,sans-serif;letter-spacing:normal}html,body{height:100%;box-sizing:border-box}body{margin:0;font-family:Roboto,Helvetica Neue,sans-serif}</style><link rel="stylesheet" href="styles.75540c58365780bb.css" media="print" onload="this.media='all'"><noscript><link rel="stylesheet" href="styles.75540c58365780bb.css"></noscript></head>
<style>.mat-typography{font:400 14px/20px Roboto,Helvetica Neue,sans-serif;letter-spacing:normal}html,body{height:100%;box-sizing:border-box}body{margin:0;font-family:Roboto,Helvetica Neue,sans-serif}</style><link rel="stylesheet" href="styles.263d22bb586b0e9e.css" media="print" onload="this.media='all'"><noscript><link rel="stylesheet" href="styles.263d22bb586b0e9e.css"></noscript></head>
<body class="mat-typography">
<app-root></app-root>
<script src="runtime.7cb62255c16cf7ce.js" type="module"></script><script src="polyfills.b6adefa6020709e6.js" type="module"></script><script src="main.48826072ab7c0c2c.js" type="module"></script>
<script src="runtime.7cb62255c16cf7ce.js" type="module"></script><script src="polyfills.ffb5811980c12988.js" type="module"></script><script src="main.019eda219eff2714.js" type="module"></script>

</body></html>
1 change: 1 addition & 0 deletions ui/dist/ui/main.019eda219eff2714.js
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion ui/dist/ui/main.48826072ab7c0c2c.js
View file

This file was deleted.

1 change: 0 additions & 1 deletion ui/dist/ui/polyfills.b6adefa6020709e6.js
View file

This file was deleted.

1 change: 1 addition & 0 deletions ui/dist/ui/polyfills.ffb5811980c12988.js
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions ui/dist/ui/styles.263d22bb586b0e9e.css
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion ui/dist/ui/styles.75540c58365780bb.css
View file

This file was deleted.

112 changes: 112 additions & 0 deletions webv2/table/review_table_schema_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,118 @@ func TestReviewTableSchema(t *testing.T) {
SpDialect: constants.DIALECT_GOOGLESQL,
},
},
{
name: "Test change type success for related foreign key columns",
tableId: "t1",
payload: `
{
"UpdateCols":{
"c1": { "ToType": "STRING" }
}
}`,
statusCode: http.StatusOK,
conv: &internal.Conv{
SpSchema: map[string]ddl.CreateTable{
"t1": {
Name: "t1",
ColIds: []string{"c1"},
ColDefs: map[string]ddl.ColumnDef{
"c1": {Name: "a", Id: "c1", T: ddl.Type{Name: ddl.Int64}, NotNull: true},
},
PrimaryKeys: []ddl.IndexKey{{ColId: "c1", Desc: false}},
ForeignKeys: []ddl.Foreignkey{{Name: "fk1", ColIds: []string{"c1"}, ReferTableId: "t2", ReferColumnIds: []string{"c4"}}},
},
"t2": {
Name: "t2",
ColIds: []string{"c4"},
ColDefs: map[string]ddl.ColumnDef{
"c4": {Name: "a", Id: "c4", T: ddl.Type{Name: ddl.Int64}, NotNull: true},
},
PrimaryKeys: []ddl.IndexKey{{ColId: "c4", Desc: false}},
},
},
SrcSchema: map[string]schema.Table{
"t1": {
Name: "t1",
ColIds: []string{"c1"},
ColDefs: map[string]schema.Column{
"c1": {Name: "a", Id: "c1", Type: schema.Type{Name: "bigint", Mods: []int64{}}, NotNull: true},
},
PrimaryKeys: []schema.Key{{ColId: "c1", Desc: false}},
ForeignKeys: []schema.ForeignKey{{Name: "fk1", ColIds: []string{"c1"}, ReferTableId: "t2", ReferColumnIds: []string{"c4"}}},
},
"t2": {
Name: "t2",
ColIds: []string{"c4"},
ColDefs: map[string]schema.Column{
"c4": {Name: "a", Id: "c4", Type: schema.Type{Name: "bigint", Mods: []int64{}}, NotNull: true},
},
PrimaryKeys: []schema.Key{{ColId: "c4", Desc: false}},
},
},
Audit: internal.Audit{MigrationType: migration.MigrationData_SCHEMA_AND_DATA.Enum()},
SchemaIssues: map[string]internal.TableIssues{
"t1": {
ColumnLevelIssues: make(map[string][]internal.SchemaIssue),
},
"t2": {
ColumnLevelIssues: make(map[string][]internal.SchemaIssue),
},
},
},
expectedConv: &internal.Conv{
SpSchema: map[string]ddl.CreateTable{
"t1": {
Name: "t1",
ColIds: []string{"c1"},
ColDefs: map[string]ddl.ColumnDef{
"c1": {Name: "a", Id: "c1", T: ddl.Type{Name: ddl.String, Len: ddl.MaxLength}, NotNull: true},
},
PrimaryKeys: []ddl.IndexKey{{ColId: "c1", Desc: false}},
ForeignKeys: []ddl.Foreignkey{{Name: "fk1", ColIds: []string{"c1"}, ReferTableId: "t2", ReferColumnIds: []string{"c4"}}},
},
"t2": {
Name: "t2",
ColIds: []string{"c4"},
ColDefs: map[string]ddl.ColumnDef{
"c4": {Name: "a", Id: "c4", T: ddl.Type{Name: ddl.String, Len: ddl.MaxLength}, NotNull: true},
},
PrimaryKeys: []ddl.IndexKey{{ColId: "c4", Desc: false}},
},
},
SrcSchema: map[string]schema.Table{
"t1": {
Name: "t1",
ColIds: []string{"c1"},
ColDefs: map[string]schema.Column{
"c1": {Name: "a", Id: "c1", Type: schema.Type{Name: "bigint", Mods: []int64{}}, NotNull: true},
},
PrimaryKeys: []schema.Key{{ColId: "c1", Desc: false}},
ForeignKeys: []schema.ForeignKey{{Name: "fk1", ColIds: []string{"c1"}, ReferTableId: "t2", ReferColumnIds: []string{"c4"}}},
},
"t2": {
Name: "t2",
ColIds: []string{"c4"},
ColDefs: map[string]schema.Column{
"c4": {Name: "a", Id: "c4", Type: schema.Type{Name: "bigint", Mods: []int64{}}, NotNull: true},
},
PrimaryKeys: []schema.Key{{ColId: "c4", Desc: false}},
},
},
SchemaIssues: map[string]internal.TableIssues{
"t1": {
ColumnLevelIssues: map[string][]internal.SchemaIssue{
"c1": {internal.Widened},
},
},
"t2": {
ColumnLevelIssues: map[string][]internal.SchemaIssue{
"c4": {internal.Widened},
},
},
},
},
},
}

for _, tc := range tc {
Expand Down
Loading

0 comments on commit 142af83

Please sign in to comment.