From f6ab9382b21eb6879277a09a89a0d67e2770d70e Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Thu, 3 Oct 2019 11:17:08 -0500 Subject: [PATCH] Address repo updates (#134) * Factor out get or create address into one sql string * Factor out getChecksumAddress method in address repo * Update address repo methods to not need a receiver * Move address repository to libraries/shared --- .../shared/repository/address_repository.go | 73 +++++++++++++++++++ .../repository}/address_repository_test.go | 46 ++++++------ .../full/retriever/block_retriever.go | 8 +- .../repositories/address_repository.go | 62 ---------------- .../full_sync_receipt_repository.go | 3 +- .../header_sync_log_repository.go | 23 +++--- .../header_sync_log_repository_test.go | 7 +- .../header_sync_receipt_repository.go | 3 +- 8 files changed, 115 insertions(+), 110 deletions(-) create mode 100644 libraries/shared/repository/address_repository.go rename {pkg/datastore/postgres/repositories => libraries/shared/repository}/address_repository_test.go (76%) delete mode 100644 pkg/datastore/postgres/repositories/address_repository.go diff --git a/libraries/shared/repository/address_repository.go b/libraries/shared/repository/address_repository.go new file mode 100644 index 000000000..40f064dc8 --- /dev/null +++ b/libraries/shared/repository/address_repository.go @@ -0,0 +1,73 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/jmoiron/sqlx" + + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +const getOrCreateAddressQuery = `WITH addressId AS ( + INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id + ) + SELECT id FROM addresses WHERE address = $1 + UNION + SELECT id FROM addressId` + +func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) { + checksumAddress := getChecksumAddress(address) + + var addressId int64 + getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress) + + return addressId, getOrCreateErr +} + +func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) { + checksumAddress := getChecksumAddress(address) + + var addressId int64 + getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress) + + return addressId, getOrCreateErr +} + +func GetAddressById(db *postgres.DB, id int64) (string, error) { + var address string + getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) + return address, getErr +} + +func getChecksumAddress(address string) string { + stringAddressToCommonAddress := common.HexToAddress(address) + return stringAddressToCommonAddress.Hex() +} diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/libraries/shared/repository/address_repository_test.go similarity index 76% rename from pkg/datastore/postgres/repositories/address_repository_test.go rename to libraries/shared/repository/address_repository_test.go index 1b2b788f5..aa81aaf48 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/libraries/shared/repository/address_repository_test.go @@ -1,22 +1,23 @@ // VulcanizeDB // Copyright © 2019 Vulcanize - +// // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. - +// // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. - +// // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package repositories_test +package repository_test import ( + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "strings" "github.com/jmoiron/sqlx" @@ -24,7 +25,6 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -32,13 +32,11 @@ import ( var _ = Describe("address lookup", func() { var ( db *postgres.DB - repo repositories.AddressRepository address = fakes.FakeAddress.Hex() ) BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - repo = repositories.AddressRepository{} }) AfterEach(func() { @@ -52,7 +50,7 @@ var _ = Describe("address lookup", func() { Describe("GetOrCreateAddress", func() { It("creates an address record", func() { - addressId, createErr := repo.GetOrCreateAddress(db, address) + addressId, createErr := repository.GetOrCreateAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) var actualAddress dbAddress @@ -63,10 +61,10 @@ var _ = Describe("address lookup", func() { }) It("returns the existing record id if the address already exists", func() { - createId, createErr := repo.GetOrCreateAddress(db, address) + createId, createErr := repository.GetOrCreateAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) - getId, getErr := repo.GetOrCreateAddress(db, address) + getId, getErr := repository.GetOrCreateAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) var addressCount int @@ -78,20 +76,20 @@ var _ = Describe("address lookup", func() { It("gets upper-cased addresses", func() { upperAddress := strings.ToUpper(address) - upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress) + upperAddressId, createErr := repository.GetOrCreateAddress(db, upperAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address) + mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) Expect(upperAddressId).To(Equal(mixedCaseAddressId)) }) It("gets lower-cased addresses", func() { lowerAddress := strings.ToLower(address) - upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress) + upperAddressId, createErr := repository.GetOrCreateAddress(db, lowerAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address) + mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) Expect(upperAddressId).To(Equal(mixedCaseAddressId)) }) @@ -112,7 +110,7 @@ var _ = Describe("address lookup", func() { }) It("creates an address record", func() { - addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address) + addressId, createErr := repository.GetOrCreateAddressInTransaction(tx, address) Expect(createErr).NotTo(HaveOccurred()) commitErr := tx.Commit() Expect(commitErr).NotTo(HaveOccurred()) @@ -125,10 +123,10 @@ var _ = Describe("address lookup", func() { }) It("returns the existing record id if the address already exists", func() { - _, createErr := repo.GetOrCreateAddressInTransaction(tx, address) + _, createErr := repository.GetOrCreateAddressInTransaction(tx, address) Expect(createErr).NotTo(HaveOccurred()) - _, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + _, getErr := repository.GetOrCreateAddressInTransaction(tx, address) Expect(getErr).NotTo(HaveOccurred()) tx.Commit() @@ -139,10 +137,10 @@ var _ = Describe("address lookup", func() { It("gets upper-cased addresses", func() { upperAddress := strings.ToUpper(address) - upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress) + upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, upperAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address) Expect(getErr).NotTo(HaveOccurred()) tx.Commit() @@ -151,10 +149,10 @@ var _ = Describe("address lookup", func() { It("gets lower-cased addresses", func() { lowerAddress := strings.ToLower(address) - upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress) + upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, lowerAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address) Expect(getErr).NotTo(HaveOccurred()) tx.Commit() @@ -164,16 +162,16 @@ var _ = Describe("address lookup", func() { Describe("GetAddressById", func() { It("gets and address by it's id", func() { - addressId, createErr := repo.GetOrCreateAddress(db, address) + addressId, createErr := repository.GetOrCreateAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) - actualAddress, getErr := repo.GetAddressById(db, addressId) + actualAddress, getErr := repository.GetAddressById(db, addressId) Expect(getErr).NotTo(HaveOccurred()) Expect(actualAddress).To(Equal(address)) }) It("returns an error if the id doesn't exist", func() { - _, getErr := repo.GetAddressById(db, 0) + _, getErr := repository.GetAddressById(db, 0) Expect(getErr).To(HaveOccurred()) Expect(getErr).To(MatchError("sql: no rows in result set")) }) diff --git a/pkg/contract_watcher/full/retriever/block_retriever.go b/pkg/contract_watcher/full/retriever/block_retriever.go index 8d720263e..24c410031 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever.go +++ b/pkg/contract_watcher/full/retriever/block_retriever.go @@ -18,8 +18,8 @@ package retriever import ( "database/sql" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) // Block retriever is used to retrieve the first block for a given contract and the most recent block @@ -55,7 +55,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { var firstBlock int64 - addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr) + addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr) if getAddressErr != nil { return firstBlock, getAddressErr } @@ -72,10 +72,6 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in return firstBlock, err } -func addressRepository() repositories.AddressRepository { - return repositories.AddressRepository{} -} - // In which case this servers as a heuristic to find the first block by finding the first contract event log func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) { var firstBlock int diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go deleted file mode 100644 index 7cf709d2a..000000000 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ /dev/null @@ -1,62 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package repositories - -import ( - "database/sql" - - "github.com/ethereum/go-ethereum/common" - "github.com/jmoiron/sqlx" - - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" -) - -type AddressRepository struct{} - -func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) { - stringAddressToCommonAddress := common.HexToAddress(address) - hexAddress := stringAddressToCommonAddress.Hex() - - var addressId int64 - getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) - if getErr == sql.ErrNoRows { - insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) - return addressId, insertErr - } - - return addressId, getErr -} - -func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) { - stringAddressToCommonAddress := common.HexToAddress(address) - hexAddress := stringAddressToCommonAddress.Hex() - - var addressId int64 - getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) - if getErr == sql.ErrNoRows { - insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) - return addressId, insertErr - } - - return addressId, getErr -} - -func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) { - var address string - getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) - return address, getErr -} diff --git a/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go b/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go index 0932dd3fb..d6c6ba196 100644 --- a/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go +++ b/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -84,7 +85,7 @@ func createLogs(logs []core.FullSyncLog, receiptId int64, tx *sqlx.Tx) error { func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { var receiptId int64 - addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) + addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) if getAddressErr != nil { logrus.Error("createReceipt: Error getting address id: ", getAddressErr) return receiptId, getAddressErr diff --git a/pkg/datastore/postgres/repositories/header_sync_log_repository.go b/pkg/datastore/postgres/repositories/header_sync_log_repository.go index 90aceab51..98dae0818 100644 --- a/pkg/datastore/postgres/repositories/header_sync_log_repository.go +++ b/pkg/datastore/postgres/repositories/header_sync_log_repository.go @@ -22,6 +22,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/lib/pq" "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -31,14 +32,12 @@ const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING` type HeaderSyncLogRepository struct { - db *postgres.DB - addressRepository AddressRepository + db *postgres.DB } func NewHeaderSyncLogRepository(db *postgres.DB) HeaderSyncLogRepository { return HeaderSyncLogRepository{ - db: db, - addressRepository: AddressRepository{}, + db: db, } } @@ -57,8 +56,8 @@ type headerSyncLog struct { Raw []byte } -func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) { - rows, queryErr := repository.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`) +func (repo HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) { + rows, queryErr := repo.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`) if queryErr != nil { return nil, queryErr } @@ -74,7 +73,7 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co for _, topic := range rawLog.Topics { logTopics = append(logTopics, common.BytesToHash(topic)) } - address, addrErr := repository.addressRepository.GetAddressById(repository.db, rawLog.Address) + address, addrErr := repository.GetAddressById(repo.db, rawLog.Address) if addrErr != nil { return nil, addrErr } @@ -104,13 +103,13 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co return results, nil } -func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error { - tx, txErr := repository.db.Beginx() +func (repo HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error { + tx, txErr := repo.db.Beginx() if txErr != nil { return txErr } for _, log := range logs { - err := repository.insertLog(headerID, log, tx) + err := repo.insertLog(headerID, log, tx) if err != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { @@ -122,13 +121,13 @@ func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, l return tx.Commit() } -func (repository HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error { +func (repo HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error { topics := buildTopics(log) raw, jsonErr := log.MarshalJSON() if jsonErr != nil { return jsonErr } - addressID, addrErr := repository.addressRepository.GetOrCreateAddressInTransaction(tx, log.Address.Hex()) + addressID, addrErr := repository.GetOrCreateAddressInTransaction(tx, log.Address.Hex()) if addrErr != nil { return addrErr } diff --git a/pkg/datastore/postgres/repositories/header_sync_log_repository_test.go b/pkg/datastore/postgres/repositories/header_sync_log_repository_test.go index 5cd062a38..a6f2d0d77 100644 --- a/pkg/datastore/postgres/repositories/header_sync_log_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_sync_log_repository_test.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + repository2 "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/libraries/shared/test_data" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -79,8 +80,7 @@ var _ = Describe("Header sync log repository", func() { Expect(lookupErr).NotTo(HaveOccurred()) Expect(dbLog.ID).NotTo(BeZero()) Expect(dbLog.HeaderID).To(Equal(headerID)) - addressRepository := repositories.AddressRepository{} - actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address) + actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address) Expect(addressErr).NotTo(HaveOccurred()) Expect(actualAddress).To(Equal(log.Address.Hex())) Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes())) @@ -128,8 +128,7 @@ var _ = Describe("Header sync log repository", func() { logTopics = append(logTopics, common.BytesToHash(topic)) } - addressRepository := repositories.AddressRepository{} - actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address) + actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address) Expect(addressErr).NotTo(HaveOccurred()) reconstructedLog := types.Log{ Address: common.HexToAddress(actualAddress), diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go index 16b876445..13029045c 100644 --- a/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go @@ -19,6 +19,7 @@ package repositories import ( "github.com/ethereum/go-ethereum/log" "github.com/jmoiron/sqlx" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/core" ) @@ -26,7 +27,7 @@ type HeaderSyncReceiptRepository struct{} func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { var receiptId int64 - addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) + addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) if getAddressErr != nil { log.Error("createReceipt: Error getting address id: ", getAddressErr) return receiptId, getAddressErr