diff --git a/docs/index.md b/docs/index.md index 0f84e4c..672b600 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,6 +63,7 @@ better alternative. - `access_token` (String, Sensitive) The access token for the Retool API - `host` (String) The host of the Retool instance, organization or Space, e.g. 'example.retool.com' +- `requests_per_minute` (Number) The number of requests per minute to allow to the Retool API. Set to 45 by default. Set to -1 to disable rate limiting. - `scheme` (String) The scheme of the Retool instance, e.g. 'https' ## Environment Variables @@ -112,3 +113,27 @@ RETOOL_SCHEME="https" \ RETOOL_ACCESS_TOKEN="your-access-token" \ terraform plan ``` + +## Rate limiting +Retool API has rate limits. In order to avoid hitting the rate limits, Retool Terraform provider is configured to limit requests to the API to 45 requests per minute. +This might be too slow for complex Retool configurations with a lot of folders and permission groups. To increase the rate limit, you can do the following: + +- Disable rate limiting on your self-hosted Retool instance by setting `DISABLE_RATE_LIMIT` environment variable to `true`. + +- Increase the rate limit on your self-hosted Retool instance by setting `API_CALLS_PER_MIN` environment variable higher than its default value of 300. + +Once you've increased the rate limit, you can increase the `requests_per_minute` parameter in the provider configuration. + +```terraform +provider "retool" { + requests_per_minute = 100 +} +``` + +Or you can disable rate limiting in the provider completely by setting `requests_per_minute` to `-1`. + +```terraform +provider "retool" { + requests_per_minute = -1 +} +``` diff --git a/examples/provider/provider_with_rate_limit.tf b/examples/provider/provider_with_rate_limit.tf new file mode 100644 index 0000000..f052dc3 --- /dev/null +++ b/examples/provider/provider_with_rate_limit.tf @@ -0,0 +1,3 @@ +provider "retool" { + requests_per_minute = 100 +} diff --git a/examples/provider/provider_without_rate_limit.tf b/examples/provider/provider_without_rate_limit.tf new file mode 100644 index 0000000..922a1e9 --- /dev/null +++ b/examples/provider/provider_without_rate_limit.tf @@ -0,0 +1,3 @@ +provider "retool" { + requests_per_minute = -1 +} diff --git a/go.mod b/go.mod index 0d291fb..5a3b8a5 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,14 @@ go 1.22.3 require ( github.com/hashicorp/terraform-plugin-docs v0.19.4 - github.com/hashicorp/terraform-plugin-framework v1.9.0 + github.com/hashicorp/terraform-plugin-framework v1.11.0 github.com/hashicorp/terraform-plugin-framework-validators v0.12.0 github.com/hashicorp/terraform-plugin-go v0.23.0 github.com/hashicorp/terraform-plugin-log v0.9.0 github.com/hashicorp/terraform-plugin-testing v1.8.0 github.com/stretchr/testify v1.8.2 golang.org/x/mod v0.17.0 + golang.org/x/time v0.6.0 gopkg.in/dnaeon/go-vcr.v3 v3.2.0 ) diff --git a/go.sum b/go.sum index 3ac9358..4b2d2f6 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= github.com/hashicorp/terraform-plugin-docs v0.19.4 h1:G3Bgo7J22OMtegIgn8Cd/CaSeyEljqjH3G39w28JK4c= github.com/hashicorp/terraform-plugin-docs v0.19.4/go.mod h1:4pLASsatTmRynVzsjEhbXZ6s7xBlUw/2Kt0zfrq8HxA= -github.com/hashicorp/terraform-plugin-framework v1.9.0 h1:caLcDoxiRucNi2hk8+j3kJwkKfvHznubyFsJMWfZqKU= -github.com/hashicorp/terraform-plugin-framework v1.9.0/go.mod h1:qBXLDn69kM97NNVi/MQ9qgd1uWWsVftGSnygYG1tImM= +github.com/hashicorp/terraform-plugin-framework v1.11.0 h1:M7+9zBArexHFXDx/pKTxjE6n/2UCXY6b8FIq9ZYhwfE= +github.com/hashicorp/terraform-plugin-framework v1.11.0/go.mod h1:qBXLDn69kM97NNVi/MQ9qgd1uWWsVftGSnygYG1tImM= github.com/hashicorp/terraform-plugin-framework-validators v0.12.0 h1:HOjBuMbOEzl7snOdOoUfE2Jgeto6JOjLVQ39Ls2nksc= github.com/hashicorp/terraform-plugin-framework-validators v0.12.0/go.mod h1:jfHGE/gzjxYz6XoUwi/aYiiKrJDeutQNUtGQXkaHklg= github.com/hashicorp/terraform-plugin-go v0.23.0 h1:AALVuU1gD1kPb48aPQUjug9Ir/125t+AAurhqphJ2Co= @@ -257,6 +257,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 909bff1..04ccd1c 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "os" + "time" "github.com/hashicorp/terraform-plugin-framework/datasource" "github.com/hashicorp/terraform-plugin-framework/path" @@ -64,9 +65,10 @@ type retoolProvider struct { } type retoolProviderModel struct { - Host types.String `tfsdk:"host"` - Scheme types.String `tfsdk:"scheme"` - AccessToken types.String `tfsdk:"access_token"` + Host types.String `tfsdk:"host"` + Scheme types.String `tfsdk:"scheme"` + AccessToken types.String `tfsdk:"access_token"` + RequestsPerMinute types.Int32 `tfsdk:"requests_per_minute"` } // Metadata returns the provider type name. @@ -92,6 +94,10 @@ func (p *retoolProvider) Schema(_ context.Context, _ provider.SchemaRequest, res Optional: true, Sensitive: true, }, + "requests_per_minute": schema.Int32Attribute{ + Description: "The number of requests per minute to allow to the Retool API. Set to 45 by default. Set to -1 to disable rate limiting.", + Optional: true, + }, }, } } @@ -191,6 +197,11 @@ func (p *retoolProvider) Configure(ctx context.Context, req provider.ConfigureRe accessToken = config.AccessToken.ValueString() } + requestsPerMinute := 45 + if !config.RequestsPerMinute.IsNull() { + requestsPerMinute = int(config.RequestsPerMinute.ValueInt32()) + } + // If any of the expected configurations are missing, return // errors with provider-specific guidance. @@ -237,6 +248,18 @@ func (p *retoolProvider) Configure(ctx context.Context, req provider.ConfigureRe // We need this to be able to record and replay HTTP interactions in the acceptance tests. if p.httpClient != nil { clientConfig.HTTPClient = p.httpClient + } else { + clientConfig.HTTPClient = http.DefaultClient + } + + if requestsPerMinute > 0 { + currentTransport := clientConfig.HTTPClient.Transport + if currentTransport == nil { + currentTransport = http.DefaultTransport + } + // Rate-limiter is using a token bucket algorithm to limit the number of requests per minute. + // The first parameter is the rate of token replenishment, the second is the capacity of the bucket. + clientConfig.HTTPClient.Transport = utils.NewThrottledTransport(time.Duration(60000/requestsPerMinute)*time.Millisecond, requestsPerMinute, currentTransport) } clientConfig.AddDefaultHeader("Authorization", "Bearer "+accessToken) diff --git a/internal/provider/utils/throttled_transport.go b/internal/provider/utils/throttled_transport.go new file mode 100644 index 0000000..4818db8 --- /dev/null +++ b/internal/provider/utils/throttled_transport.go @@ -0,0 +1,35 @@ +package utils + +import ( + "net/http" + "time" + + "golang.org/x/time/rate" +) + +// ThrottledTransport Rate Limited HTTP Client +// Copied as-is from https://gist.github.com/zdebra/10f0e284c4672e99f0cb767298f20c11 +type ThrottledTransport struct { + roundTripperWrap http.RoundTripper + ratelimiter *rate.Limiter +} + +// Implements the RoundTripper interface. +func (c *ThrottledTransport) RoundTrip(r *http.Request) (*http.Response, error) { + err := c.ratelimiter.Wait(r.Context()) // This is a blocking call. Honors the rate limit. + if err != nil { + return nil, err + } + return c.roundTripperWrap.RoundTrip(r) +} + +// NewThrottledTransport wraps transportWrap with a rate limitter +// example usage: +// client := http.DefaultClient +// client.Transport = NewThrottledTransport(10*time.Seconds, 60, http.DefaultTransport) allows 60 requests every 10 seconds. +func NewThrottledTransport(limitPeriod time.Duration, requestCount int, transportWrap http.RoundTripper) http.RoundTripper { + return &ThrottledTransport{ + roundTripperWrap: transportWrap, + ratelimiter: rate.NewLimiter(rate.Every(limitPeriod), requestCount), + } +} diff --git a/templates/index.md.tmpl b/templates/index.md.tmpl index ae2b6a4..4c623b1 100644 --- a/templates/index.md.tmpl +++ b/templates/index.md.tmpl @@ -33,3 +33,19 @@ environment variables, respectively. ### Example Usage {{ codefile "shell" "examples/provider/usage_with_env_vars.sh" }} + +## Rate limiting +Retool API has rate limits. In order to avoid hitting the rate limits, Retool Terraform provider is configured to limit requests to the API to 45 requests per minute. +This might be too slow for complex Retool configurations with a lot of folders and permission groups. To increase the rate limit, you can do the following: + +- Disable rate limiting on your self-hosted Retool instance by setting `DISABLE_RATE_LIMIT` environment variable to `true`. + +- Increase the rate limit on your self-hosted Retool instance by setting `API_CALLS_PER_MIN` environment variable higher than its default value of 300. + +Once you've increased the rate limit, you can increase the `requests_per_minute` parameter in the provider configuration. + +{{ tffile "examples/provider/provider_with_rate_limit.tf" }} + +Or you can disable rate limiting in the provider completely by setting `requests_per_minute` to `-1`. + +{{ tffile "examples/provider/provider_without_rate_limit.tf" }}