Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integration test for Foreign Key Action Migration Feature #874

Merged
merged 6 commits into from
Jul 22, 2024
Merged

Integration test for Foreign Key Action Migration Feature #874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
43cfeb3
Integration testing for FK Actions
aasthabharill-google Jul 18, 2024
5326862
minor change
aasthabharill-google Jul 18, 2024
35ae119
Merge branch 'master' into integration-test-fkactions
aasthabharill Jul 18, 2024
82b3e8a
resolving shreya's review
aasthabharill-google Jul 19, 2024
6d6cd9b
Merge branch 'integration-test-fkactions' of https://github.com/aasth…
aasthabharill-google Jul 19, 2024
a901228
Merge branch 'master' into integration-test-fkactions
aasthabharill Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/integration-tests-against-emulator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ jobs:
MYSQLPORT: 3306
MYSQLUSER: root
MYSQLDATABASE: test_interleave_table_data
MYSQLDB_FKACTION: test_foreign_key_action_data
MYSQLPWD: root

# set DynamoDB related environment variables
Expand Down Expand Up @@ -110,6 +111,7 @@ jobs:
- run: mysql --version
- run: mysql -v -P 3306 --protocol=tcp -u root -proot test < test_data/mysqldump.test.out
- run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_interleave_dump.test.out
- run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_foreignkeyaction_dump.test.out

# init sql server with test_data
- name: Install sqlcmd required for loading .sql files
Expand Down
96 changes: 96 additions & 0 deletions test_data/mysql_foreignkeyaction_dump.test.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
-- MySQL dump 10.17 Distrib 10.3.23-MariaDB, for debian-linux-gnu (x86_64)
--
-- Host: 127.0.0.1 Database: cart
-- ------------------------------------------------------
-- Server version 5.7.25-google-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;


CREATE DATABASE test_foreign_key_action_data;
USE test_foreign_key_action_data;

--
-- Table structure for table `cart`
--

DROP TABLE IF EXISTS `cart`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `cart` (
`user_id` varchar(20) NOT NULL,
`product_id` varchar(20) NOT NULL,
`quantity` bigint(20) DEFAULT NULL,
`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`,`product_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

CREATE INDEX idx ON `cart` (`quantity`);

--
-- Dumping data for table `cart`
--

LOCK TABLES `cart` WRITE;
/*!40000 ALTER TABLE `cart` DISABLE KEYS */;
INSERT INTO `cart` VALUES ('901e-a6cfc2b502dc','abc-123',1,'2020-07-20 05:10:26'),('901e-a6cfc2b502dc','axd-673',2,'2020-07-20 05:10:43'),('a86b-82493320a775','zxi-631',5,'2020-07-20 05:10:46');
/*!40000 ALTER TABLE `cart` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `products`
--

DROP TABLE IF EXISTS `products`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `products` (
`product_id` varchar(20) NOT NULL,
`description` varchar(1000) DEFAULT NULL,
`price` decimal(10,2) DEFAULT NULL,
`date_added` date DEFAULT NULL,
PRIMARY KEY (`product_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- The index name `IDX` differs only in case from index idx on `cart`.
-- This was added to cover more cases in our integration tests.
--
CREATE INDEX IDX ON `products` (`description`);

--
-- Dumping data for table `products`
--

LOCK TABLES `products` WRITE;
/*!40000 ALTER TABLE `products` DISABLE KEYS */;
INSERT INTO `products` VALUES ('abc-123','Blue suede shoes',141.99,'2020-06-06'),('axd-673','Antique typewriter',99.99,'2020-06-07'),('zxi-631','Glass vase',55.50,'2020-06-10');
/*!40000 ALTER TABLE `products` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Adding Foreign Key Constraint between Cart and Products
--
ALTER TABLE `cart` ADD CONSTRAINT `fk_cart_products` FOREIGN KEY (`product_id`) REFERENCES `products`(`product_id`) ON DELETE CASCADE ON UPDATE RESTRICT;


/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2020-08-10 17:40:01
29 changes: 28 additions & 1 deletion test_data/pg_dump.test.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,18 @@ CREATE TABLE public.cart (

ALTER TABLE public.cart OWNER TO postgres;

--
-- Name: products; Type: TABLE; Schema: public; Owner: postgres
--

CREATE TABLE public.products (
productid text NOT NULL,
description text,
price numeric(10, 2)
);

ALTER TABLE public.products OWNER TO postgres;

--
-- Name: test; Type: TABLE; Schema: public; Owner: postgres
--
Expand Down Expand Up @@ -86,6 +98,16 @@ OLJCESPC7Z 31ad80e3-182b-42b0-a164-b4c7ea976ce4 125
OLJCESPC7Z 17b14ec1-5a42-4087-bb3f-3ebd32bacf2a 106
\.

--
-- Data for Name: products; Type: TABLE DATA; Schema: public; Owner: postgres
--

COPY public.products (productid, description, price) FROM stdin;
1YMWWN1N4O bookshelf 45.50
OLJCESPC7Z armchair 54.99
2KJHWIUS9K sofa 125.00
0MWERIJB8H cupboard 59.99
\.

--
-- Data for Name: test; Type: TABLE DATA; Schema: public; Owner: postgres
Expand Down Expand Up @@ -147,6 +169,12 @@ ALTER TABLE ONLY public.test3
ALTER TABLE ONLY public.test
ADD CONSTRAINT test_pkey PRIMARY KEY (id);

--
-- Name: cart fk_cart_products; Type: CONSTRAINT; Schema: public; Owner: postgres
--
ALTER TABLE ONLY public.cart
ADD CONSTRAINT fk_cart_products FOREIGN KEY (productid) REFERENCES public.products (productid) ON DELETE NO ACTION ON UPDATE RESTRICT;

--
-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: cloudsqlsuperuser
--
Expand All @@ -160,4 +188,3 @@ GRANT ALL ON SCHEMA public TO PUBLIC;
--
-- PostgreSQL database dump complete
--

69 changes: 67 additions & 2 deletions testing/mysql/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,8 @@ func TestIntegration_MYSQL_SchemaAndDataSubcommand(t *testing.T) {
filePrefix := filepath.Join(tmpdir, dbName)

host, user, srcDb, password := os.Getenv("MYSQLHOST"), os.Getenv("MYSQLUSER"), os.Getenv("MYSQLDATABASE"), os.Getenv("MYSQLPWD")
envVars := common.ClearEnvVariables([]string{"MYSQLHOST", "MYSQLUSER", "MYSQLDATABASE", "MYSQLPWD"})
args := fmt.Sprintf("schema-and-data -source=%s -prefix=%s -source-profile='host=%s,user=%s,dbName=%s,password=%s' -target-profile='instance=%s,dbName=%s'", constants.MYSQL, filePrefix, host, user, srcDb, password, instanceID, dbName)
err := common.RunCommand(args, projectID)
common.RestoreEnvVariables(envVars)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -218,6 +216,44 @@ func TestIntegration_MySQLDUMP_SchemaAndDataSubcommand(t *testing.T) {
checkResults(t, dbURI, false)
}

func TestIntegration_MYSQL_ForeignKeyActionMigration(t *testing.T) {
onlyRunForEmulatorTest(t)
t.Parallel()

tmpdir := prepareIntegrationTest(t)
defer os.RemoveAll(tmpdir)

dbName := "mysql-foreignkey-actions"
dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName)
filePrefix := filepath.Join(tmpdir, dbName)

host, user, srcDb, password := os.Getenv("MYSQLHOST"), os.Getenv("MYSQLUSER"), os.Getenv("MYSQLDB_FKACTION"), os.Getenv("MYSQLPWD")
args := fmt.Sprintf("schema-and-data -source=%s -prefix=%s -source-profile='host=%s,user=%s,dbName=%s,password=%s' -target-profile='instance=%s,dbName=%s'", constants.MYSQL, filePrefix, host, user, srcDb, password, instanceID, dbName)
err := common.RunCommand(args, projectID)
if err != nil {
t.Fatal(err)
}
defer dropDatabase(t, dbURI)

checkForeignKeyActions(ctx, t, dbURI)
}

func TestIntegration_MySQLDUMP_ForeignKeyActionMigration(t *testing.T) {
onlyRunForEmulatorTest(t)
tmpdir := prepareIntegrationTest(t)
defer os.RemoveAll(tmpdir)

dbName := "test-schema-and-data"
dumpFilePath := "../../test_data/mysql_foreignkeyaction_dump.test.out"
filePrefix := filepath.Join(tmpdir, dbName)

dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName)
runSchemaAndDataSubcommand(t, dbName, dbURI, filePrefix, dumpFilePath)

defer dropDatabase(t, dbURI)
checkForeignKeyActions(ctx, t, dbURI)
}

func checkResults(t *testing.T, dbURI string, skipJson bool) {
// Make a query to check results.
client, err := spanner.NewClient(ctx, dbURI)
Expand Down Expand Up @@ -283,6 +319,35 @@ func checkJson(ctx context.Context, t *testing.T, client *spanner.Client, dbURI
assert.Equal(t, got_profile, want_profile)
}

func checkForeignKeyActions(ctx context.Context, t *testing.T, dbURI string) {
client, err := spanner.NewClient(ctx, dbURI)
if err != nil {
log.Fatal(err)
}
defer client.Close()

// Verifying that the row to be deleted exists in child - otherwise test will incorrectly pass
stmt := spanner.Statement{SQL: `SELECT * FROM cart WHERE product_id = "zxi-631"`}
iter := client.Single().Query(ctx, stmt)
defer iter.Stop()
row, _ := iter.Next()
assert.NotNil(t, row, "Expected rows with product_id \"zxi-631\" in table 'cart'")

// Deleting row from parent table in Spanner DB
mutation := spanner.Delete("products", spanner.Key{"zxi-631"})
_, err = client.Apply(ctx, []*spanner.Mutation{mutation})
if err != nil {
t.Fatalf("Failed to delete row: %v", err)
}

// Testing ON DELETE CASCADE i.e. row from child (cart) should have been automatically deleted
stmt = spanner.Statement{SQL: `SELECT * FROM cart WHERE product_id = "zxi-631"`}
iter = client.Single().Query(ctx, stmt)
defer iter.Stop()
_, err = iter.Next()
assert.Equal(t, iterator.Done, err, "Expected rows in table 'cart' with productid 'zxi-631' to be deleted")
}

func onlyRunForEmulatorTest(t *testing.T) {
if os.Getenv("SPANNER_EMULATOR_HOST") == "" {
t.Skip("Skipping tests only running against the emulator.")
Expand Down
87 changes: 85 additions & 2 deletions testing/postgres/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
"github.com/GoogleCloudPlatform/spanner-migration-tool/testing/common"
"github.com/stretchr/testify/assert"
"google.golang.org/api/iterator"
databasepb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
)
Expand Down Expand Up @@ -183,20 +184,67 @@ func TestIntegration_POSTGRES_SchemaSubcommand(t *testing.T) {

tmpdir := prepareIntegrationTest(t)
defer os.RemoveAll(tmpdir)

now := time.Now()
g := utils.GetUtilInfoImpl{}
dbName, _ := g.GetDatabaseName(constants.POSTGRES, now)
dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName)
filePrefix := filepath.Join(tmpdir, dbName)

args := fmt.Sprintf("schema -prefix %s -source=postgres -target-profile='instance=%s,dbName=%s'", filePrefix, instanceID, dbName)
err := common.RunCommand(args, projectID)
err := common.RunCommand(args, "emulator-test-project")
if err != nil {
t.Fatal(err)
}

defer dropDatabase(t, dbURI)
}

func TestIntegration_PGDUMP_ForeignKeyActionMigration(t *testing.T) {
onlyRunForEmulatorTest(t)
t.Parallel()

tmpdir := prepareIntegrationTest(t)
defer os.RemoveAll(tmpdir)

now := time.Now()
g := utils.GetUtilInfoImpl{}
dbName, _ := g.GetDatabaseName(constants.PGDUMP, now)
dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName)

dataFilepath := "../../test_data/pg_dump.test.out"
filePrefix := filepath.Join(tmpdir, dbName)

args := fmt.Sprintf("schema-and-data -prefix %s -source=postgres -target-profile='instance=test-instance,dbName=%s' < %s", filePrefix, dbName, dataFilepath)
err := common.RunCommand(args, "emulator-test-project")
if err != nil {
t.Fatal(err)
}
// Drop the database later.
defer dropDatabase(t, dbURI)

checkForeignKeyActions(ctx, t, dbURI)
}

func TestIntegration_POSTGRES_ForeignKeyActionMigration(t *testing.T) {
onlyRunForEmulatorTest(t)
t.Parallel()

tmpdir := prepareIntegrationTest(t)
defer os.RemoveAll(tmpdir)

now := time.Now()
g := utils.GetUtilInfoImpl{}
dbName, _ := g.GetDatabaseName(constants.POSTGRES, now)
dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName)
filePrefix := filepath.Join(tmpdir, dbName)

args := fmt.Sprintf("schema-and-data -prefix %s -source=postgres -target-profile='instance=%s,dbName=%s'", filePrefix, instanceID, dbName)
err := common.RunCommand(args, "emulator-test-project")
if err != nil {
t.Fatal(err)
}
defer dropDatabase(t, dbURI)
checkForeignKeyActions(ctx, t, dbURI)
}

func checkResults(t *testing.T, dbURI string) {
Expand Down Expand Up @@ -328,6 +376,41 @@ func checkArrays(ctx context.Context, t *testing.T, client *spanner.Client) {
}
}

func checkForeignKeyActions(ctx context.Context, t *testing.T, dbURI string) {
client, err := spanner.NewClient(ctx, dbURI)
if err != nil {
log.Fatal(err)
}
defer client.Close()

// Verifying that the row to be deleted exists in parent - otherwise testing would be incorrect
stmt := spanner.Statement{SQL: `SELECT * FROM products WHERE productid = '1YMWWN1N4O'`}
iter := client.Single().Query(ctx, stmt)
defer iter.Stop()
row, _ := iter.Next()
assert.NotNil(t, row, "Expected rows with product_id \"1YMWWN1N4O\" in table 'products'")

// Deleting row from parent table in Spanner DB
mutation := spanner.Delete("products", spanner.Key{"1YMWWN1N4O"})
_, err = client.Apply(ctx, []*spanner.Mutation{mutation})
if err != nil {
t.Fatalf("Failed to delete row: %v", err)
}

// Testing ON DELETE NO ACTION - row shouldn't have been deleted in parent and child
stmt = spanner.Statement{SQL: `SELECT * FROM products WHERE productid = '1YMWWN1N4O'`}
iter = client.Single().Query(ctx, stmt)
defer iter.Stop()
row, _ = iter.Next()
assert.NotNil(t, row, "Expected rows in table 'products' with productid '1YMWWN1N4O' to still exist")

stmt = spanner.Statement{SQL: `SELECT * FROM cart WHERE productid = '1YMWWN1N4O'`}
iter = client.Single().Query(ctx, stmt)
defer iter.Stop()
row, err = iter.Next()
assert.NotNil(t, row, "Expected rows in table 'cart' with productid '1YMWWN1N4O' to still exist")
}

func onlyRunForEmulatorTest(t *testing.T) {
if os.Getenv("SPANNER_EMULATOR_HOST") == "" {
t.Skip("Skipping tests only running against the emulator.")
Expand Down
Loading