Merge pull request #277 from julio-santiesteban/feature/polygon-backf…

…iller-symbol-filter

Added symbols filter to the polygon backfiller

contrib/polygon/backfill/backfiller/backfiller.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"runtime"
 	"runtime/debug"
+	"strings"
 	"time"
 
 	"code.cloudfoundry.org/bytefmt"
@@ -21,6 +22,7 @@ import (
 var (
 	dir, from, to        string
 	bars, quotes, trades bool
+	symbols              string
 	parallelism          int
 	apiKey               string
 
@@ -36,6 +38,8 @@ func init() {
 	flag.BoolVar(&bars, "bars", false, "backfill bars")
 	flag.BoolVar(&quotes, "quotes", false, "backfill quotes")
 	flag.BoolVar(&trades, "trades", false, "backfill trades")
+	flag.StringVar(&symbols, "symbols", "*",
+		"comma separated list of symbols to backfill, the default * means backfill all symbols")
 	flag.IntVar(&parallelism, "parallelism", runtime.NumCPU(), "parallelism (default NumCPU)")
 	flag.StringVar(&apiKey, "apiKey", "", "polygon API key")
 
@@ -82,32 +86,40 @@ func main() {
 		log.Fatal("[polygon] failed to parse to timestamp (%v)", err)
 	}
 
-	log.Info("[polygon] listing symbols")
-
-	resp, err := api.ListSymbols()
-	if err != nil {
-		log.Fatal("[polygon] failed to list symbols (%v)", err)
+	var symbolList []string
+	if symbols == "*" {
+		log.Info("[polygon] listing symbols")
+		resp, err := api.ListSymbols()
+		if err != nil {
+			log.Fatal("[polygon] failed to list symbols (%v)", err)
+		}
+		symbolList = make([]string, len(resp.Symbols))
+		for i, s := range resp.Symbols {
+			symbolList[i] = s.Symbol
+		}
+	} else {
+		symbolList = strings.Split(symbols, ",")
 	}
 
 	sem := make(chan struct{}, parallelism)
 
 	if bars {
 		log.Info("[polygon] backfilling bars from %v to %v", start, end)
 
-		for _, sym := range resp.Symbols {
+		for _, sym := range symbolList {
 			s := start
 			e := end
 
-			log.Info("[polygon] backfilling bars for %v", sym.Symbol)
+			log.Info("[polygon] backfilling bars for %v", sym)
 
 			for e.After(s) {
 				if calendar.Nasdaq.IsMarketDay(e) {
 					sem <- struct{}{}
 					go func(t time.Time) {
 						defer func() { <-sem }()
 
-						if err = backfill.Bars(sym.Symbol, t.Add(-24*time.Hour), t); err != nil {
-							log.Warn("[polygon] failed to backfill trades for %v (%v)", sym.Symbol, err)
+						if err = backfill.Bars(sym, t.Add(-24*time.Hour), t); err != nil {
+							log.Warn("[polygon] failed to backfill trades for %v (%v)", sym, err)
 						}
 					}(e)
 				}
@@ -119,20 +131,20 @@ func main() {
 	if quotes {
 		log.Info("[polygon] backfilling quotes from %v to %v", start, end)
 
-		for _, sym := range resp.Symbols {
+		for _, sym := range symbolList {
 			s := start
 			e := end
 
-			log.Info("[polygon] backfilling quotes for %v", sym.Symbol)
+			log.Info("[polygon] backfilling quotes for %v", sym)
 
 			for e.After(s) {
 				if calendar.Nasdaq.IsMarketDay(e) {
 					sem <- struct{}{}
 					go func(t time.Time) {
 						defer func() { <-sem }()
 
-						if err = backfill.Quotes(sym.Symbol, t.Add(-24*time.Hour), t); err != nil {
-							log.Warn("[polygon] failed to backfill quotes for %v (%v)", sym.Symbol, err)
+						if err = backfill.Quotes(sym, t.Add(-24*time.Hour), t); err != nil {
+							log.Warn("[polygon] failed to backfill quotes for %v (%v)", sym, err)
 						}
 					}(e)
 				}
@@ -144,20 +156,20 @@ func main() {
 	if trades {
 		log.Info("[polygon] backfilling trades from %v to %v", start, end)
 
-		for _, sym := range resp.Symbols {
+		for _, sym := range symbolList {
 			s := start
 			e := end
 
-			log.Info("[polygon] backfilling trades for %v", sym.Symbol)
+			log.Info("[polygon] backfilling trades for %v", sym)
 
 			for e.After(s) {
 				if calendar.Nasdaq.IsMarketDay(e) {
 					sem <- struct{}{}
 					go func(t time.Time) {
 						defer func() { <-sem }()
 
-						if err = backfill.Trades(sym.Symbol, t.Add(-24*time.Hour), t); err != nil {
-							log.Warn("[polygon] failed to backfill trades for %v (%v)", sym.Symbol, err)
+						if err = backfill.Trades(sym, t.Add(-24*time.Hour), t); err != nil {
+							log.Warn("[polygon] failed to backfill trades for %v (%v)", sym, err)
 						}
 					}(e)
 				}