Merge branch 'master' into integration-test-fkactions

.github/scripts/verify-files.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Define the path to the dist directory and index.html file
+DIST_DIR="ui/dist/ui"
+INDEX_FILE="${DIST_DIR}/index.html"
+
+# Check if index.html exists
+if [ ! -f "$INDEX_FILE" ]; then
+  echo "Error: index.html not found in $DIST_DIR"
+  exit 1
+fi
+
+# Define the patterns to search for
+declare -a patterns=("styles[^\"']*\.css" "runtime[^\"']*\.js" "polyfills[^\"']*\.js" "main[^\"']*\.js")
+
+# Flag to check if any file is missing
+MISSING_FILES=false
+
+# Check each pattern
+for pattern in "${patterns[@]}"; do
+  # Use grep to find the file reference in index.html
+  file=$(grep -oP "(?<=href=\"|src=\")$pattern(?=\")" "$INDEX_FILE" | head -n 1)
+
+  if [ -z "$file" ]; then
+    echo "Error: No file matching pattern $pattern found in $INDEX_FILE"
+    MISSING_FILES=true
+  else
+    # Check if the file exists in the dist directory
+    FILE_PATH="${DIST_DIR}/${file}"
+    if [ ! -f "$FILE_PATH" ]; then
+      echo "Error: File not found - $FILE_PATH"
+      MISSING_FILES=true
+    else
+      echo "File exists: $FILE_PATH"
+    fi
+  fi
+done
+
+# Exit with an error if any file was missing
+if [ "$MISSING_FILES" = true ]; then
+  echo "Verification failed. File mismatch issue. Please ensure that you have run ng build."
+  exit 1
+else
+  echo "All expected files are present. Verification passed."
+fi

.github/workflows/integration-tests-against-emulator.yaml

@@ -51,7 +51,7 @@ jobs:
 
     services:
       spanner_emulator:
-        image: gcr.io/cloud-spanner-emulator/emulator:1.5.18
+        image: gcr.io/cloud-spanner-emulator/emulator:1.5.19
         ports:
           - 9010:9010
           - 9020:9020

.github/workflows/test-coverage.yaml

@@ -47,3 +47,5 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
+          version: v0.6.0

.github/workflows/ui-generated-files-check.yaml

@@ -0,0 +1,34 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: ui-generated-files-check
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - 'ui/**'
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Verify files
+      run: |
+        chmod +x .github/scripts/verify-files.sh
+        .github/scripts/verify-files.sh

Makefile

@@ -5,6 +5,7 @@ ifndef $(GOPATH)
 endif
 # Build the default binary
 build:
+	cd ui/ && ng build
 	go build -o spanner-migration-tool
 # Build a static binary
 build-static:

common/constants/constants.go

@@ -102,8 +102,16 @@ const (
 	SMT_JOB_TABLE      string = "SMT_JOB"
 	SMT_RESOURCE_TABLE string = "SMT_RESOURCE"
 	// Auto Generated Keys
-	UUID     string = "UUID"
-	SEQUENCE string = "Sequence"
+	UUID           string = "UUID"
+	SEQUENCE       string = "Sequence"
+	AUTO_INCREMENT string = "Auto Increment"
 	// Default gcs path of the Dataflow template.
 	DEFAULT_TEMPLATE_PATH string = "gs://dataflow-templates/latest/flex/Cloud_Datastream_to_Spanner"
+
+	//FK Actions
+	FK_NO_ACTION   string = "NO ACTION"
+	FK_CASCADE     string = "CASCADE"
+	FK_SET_DEFAULT string = "SET DEFAULT"
+	FK_SET_NULL    string = "SET NULL"
+	FK_RESTRICT    string = "RESTRICT"
 )

docs/data-types/mysql.md

@@ -152,10 +152,13 @@ maps `UNIQUE` constraint into `UNIQUE` secondary index. Note that due to limitat
 mysqldump parser, we are not able to handle key column ordering (i.e. ASC/DESC) in
 mysqldump files. All key columns in mysqldump files will be treated as ASC.
 
+## Auto-Increment and Sequences
+
+The tool creates a new sequence for auto-increment columns and maps the auto-generation of these columns to this sequence. The sequence type is of *bit reversed positive*. Users need to set skip range and/or start with counter to avoid duplicate key errors.
+
 ## Other MySQL features
 
-MySQL has many other features we haven't discussed, including functions,
-sequences, procedures, triggers, (non-primary) indexes and views. The tool does
+MySQL has many other features we haven't discussed, including functions procedures, triggers, (non-primary) indexes and views. The tool does
 not support these and the relevant statements are dropped during schema
 conversion.
 

docs/ui/prepare-migration/end-migration.md

@@ -3,7 +3,7 @@ layout: default
 title: End Migration
 parent: Prepare Migration Page
 grand_parent: SMT UI
-nav_order: 8
+nav_order: 9
 ---
 
 # End migration

docs/ui/prepare-migration/gcs-metadata-path.md

@@ -0,0 +1,16 @@
+---
+layout: default
+title: GCS Metadata Path (Optional)
+parent: Prepare Migration Page
+grand_parent: SMT UI
+nav_order: 7
+---
+
+# GCS Metadata Path (Optional)
+{: .no_toc }
+
+{: .important }
+
+In case of minimal downtime sharded migration, for each shard, the Datastream jobs launched by the Spanner Migration Tool write data to a destination path on Google Cloud Storage. To store the metadata information such as session file, a new GCS Bucket is hence created. If the user does not wish for a new GCS Bucket to be created or wishes to name the bucket created, then the **GCS Metadata Path** Parameter can be set **optionally**.
+
+![](https://services.google.com/fh/files/misc/gcs-bucket-metadata.png)

docs/ui/prepare-migration/monitor.md

@@ -3,7 +3,7 @@ layout: default
 title: Monitoring
 parent: Prepare Migration Page
 grand_parent: SMT UI
-nav_order: 7
+nav_order: 8
 ---
 
 # Monitoring

docs/ui/schema-conv/issues.md

@@ -22,11 +22,12 @@ Spanner migration tool scans through the generated spanner schema and notifies t
 
 ## Warnings
 
-- Detection of an auto-increment key in source database because auto-increments are currently not supported in spanner
+- Detection of an auto-increment key in source database because auto-increments are currently not supported for all source databases.
 - Spanner data type consuming more storage than source data type
 - Redundant indexes
 - Addition of [synthetic primary key](../ui.md/#termsterminology) - synth_id
 - [Hotspotting](https://cloud.google.com/spanner/docs/schema-design) due to timestamp or auto-increment keys
+- Auto Increment has been converted to Sequence, set Ignore Range or Start with Counter to avoid duplicate value errors
 
 ## Suggestions
 

docs/ui/schema-conv/schema-conv.md

@@ -21,6 +21,7 @@ In the configure schema page, the user can view details of the source database a
         1. Interleave
         1. SQL
     1. Index details
+    1. Sequence Details (Only for Source DB MySql)
 1. Issues & Suggestions
 
 ![](https://services.google.com/fh/files/helpcenter/asset-lwlr5arntba.png)

docs/ui/schema-conv/spanner-draft.md

@@ -26,15 +26,15 @@ Users can view detailed information for a table by selecting it from the **Spann
 
 ### Column
 
-Column tab provides information on the columns that are a part of the selected table. It also provides the option to edit the column wherein a user can modify a column name, delete a column, change the data type of the column or modify the null property of the column. Once the user is done with required modifications, they can click on **SAVE & CONVERT **and the update would reflect in the session file and across all the components in the database.
+Column tab provides information on the columns that are a part of the selected table. It also provides the option to edit the column wherein a user can modify a column name, delete a column, change the data type of the column, add auto-generation to the column or modify the null property of the column. Once the user is done with required modifications, they can click on **SAVE & CONVERT **and the update would reflect in the session file and across all the components in the database.
 
-![](https://services.google.com/fh/files/helpcenter/asset-oo8ugjhmgd.png)
+![](https://services.google.com/fh/files/misc/column-info-edit.png)
 
 ### Primary Key
 
 Users can view and edit the primary key of a table from the primary key tab. They can remove/add a column from the primary key or change the order of columns in the primary key. Once these changes are made, the session file is updated and they can also be verified from the [SQL tab](#sql).  
 
-![](https://services.google.com/fh/files/helpcenter/asset-syrsuek8k4r.png)
+![](https://services.google.com/fh/files/misc/migration-pk.png)
 
 ### Foreign Key
 
@@ -82,4 +82,39 @@ Apart from the existing indexes for the source database, users can also add seco
 
 ![](https://services.google.com/fh/files/helpcenter/asset-6xbpmfyeplw.png)
 
-![](https://services.google.com/fh/files/helpcenter/asset-ck63akvjank.png)
+![](https://services.google.com/fh/files/helpcenter/asset-ck63akvjank.png)
+
+## Auto-Generated Columns
+*Only Supported for source database MySQL*
+
+Auto-Generated Columns populate Spanner columns automatically if no value is provided. Currently Spanner Migration Tool support the following techniques for auto-generation:
+1. UUID function: Generate a UUID (v4) as part of a table’s primary key DEFAULT expression.
+2. Bit reverse function: Map existing integer keys using the same logic as a bit-reversed sequence to avoid hotspotting.
+
+Refer to [documentation](https://cloud.google.com/spanner/docs/primary-key-default-value).
+
+### UUID
+
+The default recommendation for primary keys in Spanner is to use a Universally Unique Identifier, or UUID. Users can convert existing columns to be filled by UUID by choosing the **Edit** option in a table and under the **Auto-Generated** column choosing UUID.
+![](https://services.google.com/fh/files/misc/column-edit-uuid.png)
+
+### Sequences
+
+Spanner offers a SEQUENCE object that generates unique integers as part of a primary key DEFAULT expression. However, unlike a monotonic sequence, the values generated by a Spanner sequence are distributed uniformly and thus won’t hotspot at scale. Existing **Auto-Increment** columns will be mapped to a new Sequence.
+
+#### Create a new Sequence
+In order to add a sequence, the user needs to select the **Add Sequence** option and provide some details mandatory to create a sequence like **sequence name** and **sequence type**. An existing sequence can also be modified using the **Edit** button.
+
+<img src="https://services.google.com/fh/files/misc/sidebar-add-seq.png" width="30%" height="30%">
+
+![](https://services.google.com/fh/files/misc/add-seq-column-info.png)
+<img src="https://services.google.com/fh/files/misc/add-seq.png" width="30%" height="30%">
+
+To assign a sequence to a column, users need to select the **Edit** option in a table and under the **Auto-Generated** column choose a Sequence.
+
+![](https://services.google.com/fh/files/misc/edit-seq.png)
+
+#### Drop a Sequence
+Users can drop a sequence by selecting the sequence and clicking on **DROP SEQUENCE**. Once a sequence is dropped, all columns that used the sequence for auto-generated will have their auto-generation policy set to none.
+
+![](https://services.google.com/fh/files/misc/seq-info.png)

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
 	github.com/lib/pq v1.9.0
-	github.com/pganalyze/pg_query_go/v2 v2.2.0
+	github.com/pganalyze/pg_query_go/v5 v5.1.0
 	github.com/pingcap/tidb v1.1.0-beta.0.20230918090611-71bcc44f77a3
 	github.com/pingcap/tidb/parser v0.0.0-20230918090611-71bcc44f77a3
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
@@ -37,7 +37,7 @@ require (
 	google.golang.org/api v0.178.0
 	google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda
 	google.golang.org/grpc v1.63.2
-	google.golang.org/protobuf v1.34.1
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
@@ -62,7 +62,6 @@ require (
 
 require (
 	cloud.google.com/go/cloudsqlconn v1.5.2
-	cloud.google.com/go/compute v1.25.1 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/iam v1.1.7 // indirect
 	cloud.google.com/go/longrunning v0.5.6 // indirect
@@ -72,7 +71,6 @@ require (
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect
 	github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect
 	github.com/cockroachdb/errors v1.8.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
@@ -132,7 +130,6 @@ require (
 	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect