Feature/retry and ratelimit

examples/v2/rateAndRetry/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/FuzzyStatic/blizzard/v2"
+	"github.com/avast/retry-go"
+)
+
+var (
+	bnetID     string
+	bnetSECRET string
+)
+
+func init() {
+	bnetID = os.Getenv("BNET_ID")
+	bnetSECRET = os.Getenv("BNET_SECRET")
+	if bnetID == "" || bnetSECRET == "" {
+		log.Fatal("missing BNET_ID or BNET_SECRET")
+	}
+}
+func main() {
+	blizz := blizzard.NewClient(
+		bnetID,
+		bnetSECRET,
+		blizzard.EU,
+		blizzard.EnUS,
+		// This is the client default rate config, we allow 100 requests per second, and a burst budget of 10 requests
+		blizzard.NewRateOpt(1*time.Second/100, 10),
+		// This is the client default retry config
+		blizzard.NewRetryOpt(
+			retry.Attempts(3),
+			retry.Delay(100*time.Millisecond),
+			retry.DelayType(retry.BackOffDelay),
+			retry.MaxJitter(0),
+			retry.RetryIf(func(err error) bool {
+				switch {
+				case err.Error() == "429 Too Many Requests":
+					return true // recoverable error, retry
+				case err.Error() == "403 Forbidden":
+					return false
+				case err.Error() == "404 Not Found":
+					return false
+				default:
+					return false // We cannot retry this away
+				}
+			}),
+		),
+	)
+
+	mount, _, err := blizz.WoWMountIndex(context.TODO())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	out, err := json.MarshalIndent(mount, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(out[:]))
+}

go.mod

@@ -2,4 +2,8 @@ module github.com/FuzzyStatic/blizzard
 
 go 1.16
 
-require golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+require (
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+)

go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -9,5 +11,7 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

v2/blizzard.go

@@ -7,17 +7,26 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/FuzzyStatic/blizzard/v2/wowsearch"
+	"github.com/avast/retry-go"
+
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
+	"golang.org/x/time/rate"
 )
 
 // For testing
 var c *Client
 
+type ClientOpts interface {
+	Apply(c *Client)
+}
+
 // Client regional API URLs, locale, client ID, client secret
 type Client struct {
 	httpClient                                      *http.Client
@@ -31,6 +40,8 @@ type Client struct {
 	dynamicClassicNamespace, staticClassicNamespace string
 	region                                          Region
 	locale                                          Locale
+	retryopts                                       []retry.Option
+	ratelimiter                                     *rate.Limiter
 }
 
 // Region type
@@ -86,7 +97,7 @@ const (
 )
 
 // NewClient create new Blizzard structure. This structure will be used to acquire your access token and make API calls.
-func NewClient(clientID, clientSecret string, region Region, locale Locale) *Client {
+func NewClient(clientID, clientSecret string, region Region, locale Locale, opts ...ClientOpts) *Client {
 	var c = Client{
 		oauth: OAuth{
 			ClientID:     clientID,
@@ -102,6 +113,35 @@ func NewClient(clientID, clientSecret string, region Region, locale Locale) *Cli
 
 	c.SetRegion(region)
 
+	for _, opt := range opts {
+		opt.Apply(&c)
+	}
+
+	if c.ratelimiter == nil {
+		c.ratelimiter = rate.NewLimiter(rate.Every(1*time.Second/100), 10)
+	}
+
+	if len(c.retryopts) == 0 {
+		c.retryopts = []retry.Option{
+			retry.Attempts(3),
+			retry.Delay(100 * time.Millisecond),
+			retry.DelayType(retry.BackOffDelay),
+			retry.MaxJitter(0),
+			retry.RetryIf(func(err error) bool {
+				switch {
+				case err.Error() == "429 Too Many Requests":
+					return true // recoverable error, retry
+				case err.Error() == "403 Forbidden":
+					return false
+				case err.Error() == "404 Not Found":
+					return false
+				default:
+					return false // unhandled error
+				}
+			}),
+		}
+	}
+
 	return &c
 }
 
@@ -144,7 +184,19 @@ func (c *Client) SetRegion(region Region) {
 	}
 
 	c.cfg.TokenURL = c.oauthHost + "/oauth/token"
-	c.httpClient = c.cfg.Client(context.Background())
+
+	defaultTransport := &http.Transport{
+		Dial:                (&net.Dialer{KeepAlive: 10 * time.Second}).Dial,
+		MaxIdleConns:        6,
+		MaxIdleConnsPerHost: 2,
+	}
+
+	httpClient := &http.Client{
+		Timeout:   10 * time.Second,
+		Transport: defaultTransport,
+	}
+	ctx := context.WithValue(context.TODO(), oauth2.HTTPClient, httpClient)
+	c.httpClient = c.cfg.Client(ctx)
 }
 
 // GetOAuthHost returns the OAuth host of the client
@@ -194,13 +246,20 @@ func buildSearchParams(opts ...wowsearch.Opt) string {
 	return "?" + strings.Join(params, "&")
 }
 
+func (c *Client) getRetryOpts(ctx context.Context) []retry.Option {
+	opts := []retry.Option{
+		retry.Context(ctx),
+	}
+	return append(opts, c.retryopts...)
+}
+
 // getStructData processes simple GET request based on pathAndQuery an returns the structured data.
 func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace string, dat interface{}) (interface{}, *Header, error) {
+
 	req, err := http.NewRequestWithContext(ctx, "GET", c.apiHost+pathAndQuery, nil)
 	if err != nil {
 		return dat, nil, err
 	}
-
 	req.Header.Set("Accept", "application/json")
 
 	q := req.URL.Query()
@@ -211,10 +270,24 @@ func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace stri
 		req.Header.Set("Battlenet-Namespace", namespace)
 	}
 
-	res, err := c.httpClient.Do(req)
+	var res *http.Response
+	err = retry.Do(
+		func() (err error) {
+			if err := c.ratelimiter.Wait(ctx); err != nil {
+				return err
+			}
+			res, err = c.httpClient.Do(req)
+			if err != nil && res != nil {
+				res.Body.Close()
+			}
+			return
+		},
+		c.getRetryOpts(ctx)...,
+	)
 	if err != nil {
 		return dat, nil, err
 	}
+
 	defer res.Body.Close()
 
 	body, err := ioutil.ReadAll(res.Body)
@@ -237,6 +310,7 @@ func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace stri
 	}
 
 	return dat, header, nil
+
 }
 
 // getStructDataNoNamespace processes simple GET request based on pathAndQuery an returns the structured data.

v2/blizzard_opts.go

@@ -0,0 +1,47 @@
+package blizzard
+
+import (
+	"time"
+
+	"github.com/avast/retry-go"
+	"golang.org/x/time/rate"
+)
+
+// NewRateOpt sets the rate of which the client can do requests
+// Blizzard allows 36000 per hour or up to 100 per second
+//
+// Example:
+//
+//  blizzard.NewRateOpt(1*time.Second/100, 10)
+func NewRateOpt(rate time.Duration, burst int) ClientOpts {
+	return &RateOpt{
+		rate: rate,
+		b:    burst,
+	}
+}
+
+type RateOpt struct {
+	rate time.Duration
+	b    int
+}
+
+func (r *RateOpt) Apply(c *Client) {
+	c.ratelimiter = rate.NewLimiter(
+		rate.Every(r.rate),
+		r.b,
+	)
+}
+
+func NewRetryOpt(opts ...retry.Option) ClientOpts {
+	return &RetryOpt{
+		opts: opts,
+	}
+}
+
+type RetryOpt struct {
+	opts []retry.Option
+}
+
+func (r *RetryOpt) Apply(c *Client) {
+	c.retryopts = r.opts
+}