Moved to retrying HTTP client (#2947)

There are some situations where will hit transient network issues when
polling for Overmind to complete it's analysis. We don't want these
small issues to cancel the whole process so I've mode to a retrying HTTP
client

GitOrigin-RevId: eea19ba893a976f5012ce80d6fb126a98419ff02

Author: dylanratcliffe

cmd/auth_client.go

@@ -4,14 +4,29 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/overmindtech/cli/auth"
 	"github.com/overmindtech/cli/sdp-go"
 	"github.com/overmindtech/cli/sdp-go/sdpconnect"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
+// newRetryableHTTPClient creates a new HTTP client that uses standard retryablehttp settings
+func newRetryableHTTPClient() *http.Client {
+	retryableClient := &retryablehttp.Client{
+		HTTPClient:   otelhttp.DefaultClient,
+		RetryWaitMin: 1 * time.Second,
+		RetryWaitMax: 10 * time.Second,
+		RetryMax:     5,
+		CheckRetry:   retryablehttp.DefaultRetryPolicy,
+		Backoff:      retryablehttp.DefaultBackoff,
+	}
+	return retryableClient.StandardClient()
+}
+
 // UnauthenticatedApiKeyClient Returns an apikey client with otel instrumentation
 // but no authentication. Can only be used for ExchangeKeyForToken
 func UnauthenticatedApiKeyClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.ApiKeyServiceClient {
@@ -22,53 +37,53 @@ func UnauthenticatedApiKeyClient(ctx context.Context, oi sdp.OvermindInstance) s
 // AuthenticatedBookmarkClient Returns a bookmark client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedBookmarkClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.BookmarksServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind bookmark API")
 	return sdpconnect.NewBookmarksServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 // AuthenticatedChangesClient Returns a changes client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedChangesClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.ChangesServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind changes API")
 	return sdpconnect.NewChangesServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 // AuthenticatedConfigurationClient  Returns a config client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedConfigurationClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.ConfigurationServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind configuration API")
 	return sdpconnect.NewConfigurationServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 // AuthenticatedManagementClient Returns a management client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedManagementClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.ManagementServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind management API")
 	return sdpconnect.NewManagementServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 // AuthenticatedSnapshotsClient Returns a Snapshots client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedSnapshotsClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.SnapshotsServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind snapshot API")
 	return sdpconnect.NewSnapshotsServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 // AuthenticatedInviteClient Returns a Invite client that uses the auth
 // embedded in the context and otel instrumentation
 func AuthenticatedInviteClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.InviteServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind invite API")
 	return sdpconnect.NewInviteServiceClient(httpClient, oi.ApiUrl.String())
 }
 
 func AuthenticatedSignalsClient(ctx context.Context, oi sdp.OvermindInstance) sdpconnect.SignalServiceClient {
-	httpClient := NewAuthenticatedClient(ctx, otelhttp.DefaultClient)
+	httpClient := NewAuthenticatedClient(ctx, newRetryableHTTPClient())
 	log.WithContext(ctx).WithField("apiUrl", oi.ApiUrl).Debug("Connecting to overmind signals API")
 	return sdpconnect.NewSignalServiceClient(httpClient, oi.ApiUrl.String())
 }