From 09a768895e3e5772aa10f53081b73d7a01485b6a Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Tue, 2 Aug 2022 20:37:49 -0500 Subject: [PATCH 1/3] fix: download fed files in-memory instead of flat files This works around permission issues with the scratch container. Issue: https://github.com/moov-io/fed/issues/186 --- cmd/server/main.go | 16 +++++++++------- cmd/server/reader.go | 29 ++++++++++++++++++++--------- pkg/download/download.go | 29 +++++++++++++++-------------- pkg/download/download_test.go | 20 ++++++++++---------- 4 files changed, 54 insertions(+), 40 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 3d45e4ed..39ac2aa1 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -10,6 +10,7 @@ import ( "errors" "flag" "fmt" + "io" "net/http" "os" "os/signal" @@ -111,7 +112,11 @@ func main() { // Start our searcher searcher := &searcher{logger: logger} - if err := setupSearcher(logger, searcher, fedACHDataFile(logger), fedWireDataFile(logger)); err != nil { + + fedACHData := fedACHDataFile(logger) + fedWireData := fedWireDataFile(logger) + + if err := setupSearcher(logger, searcher, fedACHData, fedWireData); err != nil { logger.Logf("read: %v", err) os.Exit(1) } @@ -154,7 +159,7 @@ func addPingRoute(r *mux.Router) { }) } -func setupSearcher(logger log.Logger, s *searcher, achFile, wireFile *os.File) error { +func setupSearcher(logger log.Logger, s *searcher, achFile, wireFile io.Reader) error { if achFile == nil { return errors.New("missing fedach data file") } @@ -162,15 +167,12 @@ func setupSearcher(logger log.Logger, s *searcher, achFile, wireFile *os.File) e return errors.New("missing fedwire data file") } - logger.Logf("search: loading %s for ACH data", achFile.Name()) if err := s.readFEDACHData(achFile); err != nil { - return fmt.Errorf("error reading ACH file at %s: %v", achFile.Name(), err) + return fmt.Errorf("error reading ACH data: %v", err) } - logger.Logf("search: loading %s for Wire data", wireFile.Name()) if err := s.readFEDWIREData(wireFile); err != nil { - return fmt.Errorf("error reading wire file at %s: %v", wireFile.Name(), err) - + return fmt.Errorf("error reading wire data: %v", err) } return nil } diff --git a/cmd/server/reader.go b/cmd/server/reader.go index 78f08752..6954ec47 100644 --- a/cmd/server/reader.go +++ b/cmd/server/reader.go @@ -6,6 +6,7 @@ package main import ( "fmt" + "io" "os" "github.com/moov-io/base/log" @@ -13,7 +14,7 @@ import ( "github.com/moov-io/fed/pkg/download" ) -func fedACHDataFile(logger log.Logger) *os.File { +func fedACHDataFile(logger log.Logger) io.Reader { if file, err := attemptFileDownload(logger, "fedach"); file != nil { return file } else if err != nil { @@ -21,6 +22,8 @@ func fedACHDataFile(logger log.Logger) *os.File { } path := readDataFilepath("FEDACH_DATA_PATH", "./data/FedACHdir.txt") + logger.Logf("search: loading %s for ACH data", path) + file, err := os.Open(path) if err != nil { panic(fmt.Sprintf("problem opening %s: %v", path, err)) @@ -28,7 +31,7 @@ func fedACHDataFile(logger log.Logger) *os.File { return file } -func fedWireDataFile(logger log.Logger) *os.File { +func fedWireDataFile(logger log.Logger) io.Reader { if file, err := attemptFileDownload(logger, "fedwire"); file != nil { return file } else if err != nil { @@ -36,6 +39,8 @@ func fedWireDataFile(logger log.Logger) *os.File { } path := readDataFilepath("FEDWIRE_DATA_PATH", "./data/fpddir.txt") + logger.Logf("search: loading %s for Wire data", path) + file, err := os.Open(path) if err != nil { panic(fmt.Sprintf("problem opening %s: %v", path, err)) @@ -43,7 +48,7 @@ func fedWireDataFile(logger log.Logger) *os.File { return file } -func attemptFileDownload(logger log.Logger, listName string) (*os.File, error) { +func attemptFileDownload(logger log.Logger, listName string) (io.Reader, error) { routingNumber := os.Getenv("FRB_ROUTING_NUMBER") downloadCode := os.Getenv("FRB_DOWNLOAD_CODE") @@ -71,14 +76,17 @@ func readDataFilepath(env, fallback string) string { // readFEDACHData opens and reads FedACHdir.txt then runs ACHDictionary.Read() to // parse and define ACHDictionary properties -func (s *searcher) readFEDACHData(file *os.File) error { +func (s *searcher) readFEDACHData(reader io.Reader) error { if s.logger != nil { s.logger.Logf("Read of FED data") } - defer file.Close() + + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } s.ACHDictionary = fed.NewACHDictionary() - if err := s.ACHDictionary.Read(file); err != nil { + if err := s.ACHDictionary.Read(reader); err != nil { return fmt.Errorf("ERROR: reading FedACHdir.txt %v", err) } @@ -91,14 +99,17 @@ func (s *searcher) readFEDACHData(file *os.File) error { // readFEDWIREData opens and reads fpddir.txt then runs WIREDictionary.Read() to // parse and define WIREDictionary properties -func (s *searcher) readFEDWIREData(file *os.File) error { +func (s *searcher) readFEDWIREData(reader io.Reader) error { if s.logger != nil { s.logger.Logf("Read of FED data") } - defer file.Close() + + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } s.WIREDictionary = fed.NewWIREDictionary() - if err := s.WIREDictionary.Read(file); err != nil { + if err := s.WIREDictionary.Read(reader); err != nil { return fmt.Errorf("ERROR: reading fpddir.txt %v", err) } diff --git a/pkg/download/download.go b/pkg/download/download.go index 0fbb6b1d..d507b717 100644 --- a/pkg/download/download.go +++ b/pkg/download/download.go @@ -5,10 +5,10 @@ package download import ( + "bytes" "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -65,9 +65,9 @@ func init() { } } -// GetList downloads an FRB list and saves it into a temporary file. +// GetList downloads an FRB list and saves it into an io.Reader. // Example listName values: fedach, fedwire -func (c *Client) GetList(listName string) (*os.File, error) { +func (c *Client) GetList(listName string) (io.Reader, error) { where, err := url.Parse(fmt.Sprintf("https://frbservices.org/EPaymentsDirectory/directories/%s?format=json", listName)) if err != nil { return nil, fmt.Errorf("url: %v", err) @@ -85,20 +85,21 @@ func (c *Client) GetList(listName string) (*os.File, error) { if err != nil { return nil, fmt.Errorf("http get: %v", err) } - defer resp.Body.Close() + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } - out, err := ioutil.TempFile(downloadDirectory, fmt.Sprintf("%s-*", listName)) - if err != nil { - return nil, fmt.Errorf("temp file: %v", err) + // Quit if we fail to download + if resp.StatusCode >= 299 { + return nil, fmt.Errorf("unexpected http status: %d", resp.StatusCode) } - if n, err := io.Copy(out, resp.Body); n == 0 || err != nil { + + var out bytes.Buffer + if n, err := io.Copy(&out, resp.Body); n == 0 || err != nil { return nil, fmt.Errorf("copying n=%d: %v", n, err) } - if err := out.Sync(); err != nil { - return nil, fmt.Errorf("sync: %v", err) - } - if _, err := out.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("seek: %v", err) + if out.Len() > 0 { + return &out, nil } - return out, nil + return nil, nil } diff --git a/pkg/download/download_test.go b/pkg/download/download_test.go index ced3a4da..e14f08ad 100644 --- a/pkg/download/download_test.go +++ b/pkg/download/download_test.go @@ -10,6 +10,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/stretchr/testify/require" ) func TestClient__fedach(t *testing.T) { @@ -20,11 +22,10 @@ func TestClient__fedach(t *testing.T) { t.Fatal(err) } - st, err := fedach.Stat() - if err != nil { - t.Fatal(err) - } - if n := st.Size(); n < 1024 { + buf, ok := fedach.(*bytes.Buffer) + require.True(t, ok) + + if n := buf.Len(); n < 1024 { t.Errorf("unexpected size of %d bytes", n) } @@ -42,11 +43,10 @@ func TestClient__fedwire(t *testing.T) { t.Fatal(err) } - st, err := fedwire.Stat() - if err != nil { - t.Fatal(err) - } - if n := st.Size(); n < 1024 { + buf, ok := fedwire.(*bytes.Buffer) + require.True(t, ok) + + if n := buf.Len(); n < 1024 { t.Errorf("unexpected size of %d bytes", n) } From 9a92507d691d6a90cb40369aacf7c5259b593db9 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Wed, 3 Aug 2022 08:02:43 -0500 Subject: [PATCH 2/3] docs: remove DOWNLOAD_DIRECTORY --- CHANGELOG.md | 6 ++++++ README.md | 1 - pkg/download/download.go | 13 ------------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1e2dfd..f0a71ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.9.0 (Released 2022-08-03) + +IMPROVEMENTS + +- Remove `DOWNLOAD_DIRECTORY` and store downloaded files in memory. + ## v0.8.1 (Released 2022-08-02) IMPROVEMENTS diff --git a/README.md b/README.md index 69dc4b31..6084fe03 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,6 @@ PONG | `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file | `./data/fpddir.txt` | | `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files | Empty | | `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files | Empty | -| `DOWNLOAD_DIRECTORY` | Directory for saving downloaded eServices files into | OS Temp Dir | | `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` | | `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8086` | | `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9096` | diff --git a/pkg/download/download.go b/pkg/download/download.go index d507b717..f0876e11 100644 --- a/pkg/download/download.go +++ b/pkg/download/download.go @@ -11,10 +11,7 @@ import ( "io" "net/http" "net/url" - "os" "time" - - "github.com/moov-io/base/strx" ) type Client struct { @@ -55,16 +52,6 @@ func NewClient(opts *ClientOpts) (*Client, error) { }, nil } -var ( - downloadDirectory = strx.Or(os.Getenv("DOWNLOAD_DIRECTORY"), os.TempDir()) -) - -func init() { - if _, err := os.Stat(downloadDirectory); os.IsNotExist(err) { - os.MkdirAll(downloadDirectory, 0777) - } -} - // GetList downloads an FRB list and saves it into an io.Reader. // Example listName values: fedach, fedwire func (c *Client) GetList(listName string) (io.Reader, error) { From 8ce8506b2459c8160b446bc7e9698bfdc0dff989 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Wed, 3 Aug 2022 08:03:02 -0500 Subject: [PATCH 3/3] release v0.9.0 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 18d61c3e..85f4b555 100644 --- a/version.go +++ b/version.go @@ -5,4 +5,4 @@ package fed // Version is the current version -const Version = "v0.8.1" +const Version = "v0.9.0"