From 2cd6e4641a870b19be307e863a4a695a07378a2e Mon Sep 17 00:00:00 2001 From: Neil Shah Date: Sun, 10 Dec 2017 23:32:58 -0800 Subject: [PATCH 1/5] add a few more cast types using character varying --- lib/postgres_to_redshift/column.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/postgres_to_redshift/column.rb b/lib/postgres_to_redshift/column.rb index 2b3a13f..67c98fd 100644 --- a/lib/postgres_to_redshift/column.rb +++ b/lib/postgres_to_redshift/column.rb @@ -53,6 +53,9 @@ class PostgresToRedshift::Column "bytea" => "CHARACTER VARYING(65535)", "money" => "DECIMAL(19,2)", "oid" => "CHARACTER VARYING(65535)", + "uuid" => "CHARACTER VARYING(65535)", + "inet" => "CHARACTER VARYING(65535)", + "time without time zone" => "CHARACTER VARYING(65535)", "ARRAY" => "CHARACTER VARYING(65535)", "USER-DEFINED" => "CHARACTER VARYING(65535)", } From e8ab77c9b5ef4fd7b9ead9afc46b964361bab418 Mon Sep 17 00:00:00 2001 From: Neil Shah Date: Tue, 12 Dec 2017 16:44:46 -0800 Subject: [PATCH 2/5] updated gemfile.lock --- Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c85f203..d740219 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,20 +3,20 @@ PATH specs: postgres_to_redshift (0.1.2) aws-sdk-v1 (~> 1.54) - pg (~> 0.17.0) + pg (~> 0.18.0) GEM remote: https://rubygems.org/ specs: - aws-sdk-v1 (1.66.0) + aws-sdk-v1 (1.67.0) json (~> 1.4) - nokogiri (>= 1.4.4) + nokogiri (~> 1) diff-lcs (1.2.5) - json (1.8.3) - mini_portile (0.6.2) - nokogiri (1.6.6.2) - mini_portile (~> 0.6.0) - pg (0.17.1) + json (1.8.6) + mini_portile2 (2.3.0) + nokogiri (1.8.1) + mini_portile2 (~> 2.3.0) + pg (0.18.4) rake (10.4.2) rspec (3.2.0) rspec-core (~> 3.2.0) @@ -42,4 +42,4 @@ DEPENDENCIES rspec BUNDLED WITH - 1.10.5 + 1.16.0 From 054f9017b477a7e553aa5b5a35c4706048f1a752 Mon Sep 17 00:00:00 2001 From: Neil Shah Date: Thu, 14 Dec 2017 21:57:24 -0800 Subject: [PATCH 3/5] add retry logic --- lib/postgres_to_redshift.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/postgres_to_redshift.rb b/lib/postgres_to_redshift.rb index 65c9251..4290656 100644 --- a/lib/postgres_to_redshift.rb +++ b/lib/postgres_to_redshift.rb @@ -129,19 +129,23 @@ def upload_table(table, buffer, chunk) end def import_table(table) + retries ||= 0 puts "Importing #{table.target_table_name}" schema = self.class.schema - - target_connection.exec("DROP TABLE IF EXISTS #{schema}.#{table.target_table_name}_updating") + begin + target_connection.exec("DROP TABLE IF EXISTS #{schema}.#{table.target_table_name}_updating") - target_connection.exec("BEGIN;") + target_connection.exec("BEGIN;") - target_connection.exec("ALTER TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} RENAME TO #{table.target_table_name}_updating") + target_connection.exec("ALTER TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} RENAME TO #{table.target_table_name}_updating") - target_connection.exec("CREATE TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") + target_connection.exec("CREATE TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") - target_connection.exec("COPY #{schema}.#{target_connection.quote_ident(table.target_table_name)} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';") + target_connection.exec("COPY #{schema}.#{target_connection.quote_ident(table.target_table_name)} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';") - target_connection.exec("COMMIT;") + target_connection.exec("COMMIT;") + rescue StandardError => e + retry if (retries += 1) < 3 + end end end From 158ff3845b63a0b4240c94b97ca6f1f819c5ac1f Mon Sep 17 00:00:00 2001 From: Neil Shah Date: Thu, 14 Dec 2017 22:17:15 -0800 Subject: [PATCH 4/5] Revert "add retry logic" This reverts commit 054f9017b477a7e553aa5b5a35c4706048f1a752. --- lib/postgres_to_redshift.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/postgres_to_redshift.rb b/lib/postgres_to_redshift.rb index 4290656..65c9251 100644 --- a/lib/postgres_to_redshift.rb +++ b/lib/postgres_to_redshift.rb @@ -129,23 +129,19 @@ def upload_table(table, buffer, chunk) end def import_table(table) - retries ||= 0 puts "Importing #{table.target_table_name}" schema = self.class.schema - begin - target_connection.exec("DROP TABLE IF EXISTS #{schema}.#{table.target_table_name}_updating") + + target_connection.exec("DROP TABLE IF EXISTS #{schema}.#{table.target_table_name}_updating") - target_connection.exec("BEGIN;") + target_connection.exec("BEGIN;") - target_connection.exec("ALTER TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} RENAME TO #{table.target_table_name}_updating") + target_connection.exec("ALTER TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} RENAME TO #{table.target_table_name}_updating") - target_connection.exec("CREATE TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") + target_connection.exec("CREATE TABLE #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") - target_connection.exec("COPY #{schema}.#{target_connection.quote_ident(table.target_table_name)} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';") + target_connection.exec("COPY #{schema}.#{target_connection.quote_ident(table.target_table_name)} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';") - target_connection.exec("COMMIT;") - rescue StandardError => e - retry if (retries += 1) < 3 - end + target_connection.exec("COMMIT;") end end From 98880f140acd9d1d9e81b17309dfeaca24284218 Mon Sep 17 00:00:00 2001 From: Neil Shah Date: Thu, 14 Dec 2017 22:28:42 -0800 Subject: [PATCH 5/5] try with retries --- lib/postgres_to_redshift.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/postgres_to_redshift.rb b/lib/postgres_to_redshift.rb index 65c9251..3f0fb5f 100644 --- a/lib/postgres_to_redshift.rb +++ b/lib/postgres_to_redshift.rb @@ -22,11 +22,17 @@ def self.update_tables update_tables = PostgresToRedshift.new update_tables.tables.each do |table| - target_connection.exec("CREATE TABLE IF NOT EXISTS #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") + begin + retries ||= 3 + target_connection.exec("CREATE TABLE IF NOT EXISTS #{schema}.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})") - update_tables.copy_table(table) + update_tables.copy_table(table) - update_tables.import_table(table) + update_tables.import_table(table) + rescue StandardError => e + retry unless retries += 1 < 3 + next + end end end