From 7e9cdd8b07bb6e8e519c0c5d3e3e57348978e84d Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 28 Jul 2023 15:55:39 +0800 Subject: [PATCH] fix(aliyundrive_open): fail limit on concurrently call (#4851) --- drivers/aliyundrive_open/driver.go | 11 ++- drivers/aliyundrive_share/driver.go | 11 ++- go.mod | 3 +- go.sum | 5 ++ pkg/utils/fn_limiter.go | 114 ---------------------------- pkg/utils/fn_limiter_test.go | 59 -------------- 6 files changed, 25 insertions(+), 178 deletions(-) delete mode 100644 pkg/utils/fn_limiter.go delete mode 100644 pkg/utils/fn_limiter_test.go diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 680225ae885..90e2fc7e495 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -39,8 +40,14 @@ func (d *AliyundriveOpen) Init(ctx context.Context) error { return err } d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() - d.limitList = utils.LimitRateCtx(d.list, time.Second/3) - d.limitLink = utils.LimitRateCtx(d.link, time.Second+time.Millisecond*2) + d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ + Limit: 4, + Bucket: 1, + }) + d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ + Limit: 1, + Bucket: 1, + }) return nil } diff --git a/drivers/aliyundrive_share/driver.go b/drivers/aliyundrive_share/driver.go index 61f1207e3b8..2e042ceef35 100644 --- a/drivers/aliyundrive_share/driver.go +++ b/drivers/aliyundrive_share/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -52,8 +53,14 @@ func (d *AliyundriveShare) Init(ctx context.Context) error { log.Errorf("%+v", err) } }) - d.limitList = utils.LimitRateCtx(d.list, time.Second/4) - d.limitLink = utils.LimitRateCtx(d.link, time.Second) + d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ + Limit: 4, + Bucket: 1, + }) + d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ + Limit: 1, + Bucket: 1, + }) return nil } diff --git a/go.mod b/go.mod index 440df1bd19e..24e0201534e 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible // indirect github.com/andreburgaud/crypt2go v1.1.0 // indirect @@ -135,7 +136,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 745a5fa9b50..163570e48e1 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/SheltonZhu/115driver v1.0.14 h1:uW3dl8J9KDMw+3gPxQdhTysoGhw0/uI1484GT github.com/SheltonZhu/115driver v1.0.14/go.mod h1:00ixivHH5HqDj4S7kAWbkuUrjtsJTxc7cGv5RMw3RVs= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= +github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= +github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -210,6 +212,7 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -394,6 +397,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/utils/fn_limiter.go b/pkg/utils/fn_limiter.go deleted file mode 100644 index 6f9d461fa69..00000000000 --- a/pkg/utils/fn_limiter.go +++ /dev/null @@ -1,114 +0,0 @@ -package utils - -import ( - "context" - "reflect" - "time" -) - -func LimitRateReflect(f interface{}, interval time.Duration) func(...interface{}) []interface{} { - // Use closures to save the time of the last function call - var lastCall time.Time - - fValue := reflect.ValueOf(f) - fType := fValue.Type() - - if fType.Kind() != reflect.Func { - panic("f must be a function") - } - - //if fType.NumOut() == 0 { - // panic("f must have at least one output parameter") - //} - - outCount := fType.NumOut() - outTypes := make([]reflect.Type, outCount) - - for i := 0; i < outCount; i++ { - outTypes[i] = fType.Out(i) - } - - // Returns a new function, which is used to limit the function to be called only once at a specified time interval - return func(args ...interface{}) []interface{} { - // Calculate the time interval since the last function call - elapsed := time.Since(lastCall) - // If the interval is less than the specified time, wait for the remaining time - if elapsed < interval { - time.Sleep(interval - elapsed) - } - // Update the time of the last function call - lastCall = time.Now() - - inCount := fType.NumIn() - in := make([]reflect.Value, inCount) - - if len(args) != inCount { - panic("wrong number of arguments") - } - - for i := 0; i < inCount; i++ { - in[i] = reflect.ValueOf(args[i]) - } - - out := fValue.Call(in) - - if len(out) != outCount { - panic("function returned wrong number of values") - } - - result := make([]interface{}, outCount) - - for i := 0; i < outCount; i++ { - result[i] = out[i].Interface() - } - - return result - } -} - -type Fn[T any, R any] func(T) (R, error) -type FnCtx[T any, R any] func(context.Context, T) (R, error) - -func LimitRate[T any, R any](f Fn[T, R], interval time.Duration) Fn[T, R] { - // Use closures to save the time of the last function call - var lastCall time.Time - // Returns a new function, which is used to limit the function to be called only once at a specified time interval - return func(t T) (R, error) { - // Calculate the time interval since the last function call - elapsed := time.Since(lastCall) - // If the interval is less than the specified time, wait for the remaining time - if elapsed < interval { - time.Sleep(interval - elapsed) - } - // Update the time of the last function call - lastCall = time.Now() - // Execute the function that needs to be limited - return f(t) - } -} - -func LimitRateCtx[T any, R any](f FnCtx[T, R], interval time.Duration) FnCtx[T, R] { - // Use closures to save the time of the last function call - var lastCall time.Time - // Returns a new function, which is used to limit the function to be called only once at a specified time interval - return func(ctx context.Context, t T) (R, error) { - // Calculate the time interval since the last function call - elapsed := time.Since(lastCall) - // If the interval is less than the specified time, wait for the remaining time - if elapsed < interval { - t := time.NewTimer(interval - elapsed) - select { - case <-ctx.Done(): - t.Stop() - var zero R - return zero, ctx.Err() - case <-t.C: - - } - } - // Update the time of the last function call - lastCall = time.Now() - // Execute the function that needs to be limited - return f(ctx, t) - } -} diff --git a/pkg/utils/fn_limiter_test.go b/pkg/utils/fn_limiter_test.go deleted file mode 100644 index 599cee95755..00000000000 --- a/pkg/utils/fn_limiter_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package utils_test - -import ( - "context" - "testing" - "time" - - "github.com/alist-org/alist/v3/pkg/utils" -) - -func myFunction(a int) (int, error) { - // do something - return a + 1, nil -} - -func TestLimitRate(t *testing.T) { - myLimitedFunction := utils.LimitRate(myFunction, time.Second) - result, _ := myLimitedFunction(1) - t.Log(result) // Output: 2 - result, _ = myLimitedFunction(2) - t.Log(result) // Output: 3 -} - -type Test struct { - limitFn func(string) (string, error) -} - -func (t *Test) myFunction(a string) (string, error) { - // do something - return a + " world", nil -} - -func TestLimitRateStruct(t *testing.T) { - test := &Test{} - test.limitFn = utils.LimitRate(test.myFunction, time.Second) - result, _ := test.limitFn("hello") - t.Log(result) // Output: hello world - result, _ = test.limitFn("hi") - t.Log(result) // Output: hi world -} - -func myFunctionCtx(ctx context.Context, a int) (int, error) { - // do something - return a + 1, nil -} -func TestLimitRateCtx(t *testing.T) { - myLimitedFunction := utils.LimitRateCtx(myFunctionCtx, time.Second) - result, _ := myLimitedFunction(context.Background(), 1) - t.Log(result) // Output: 2 - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(500 * time.Millisecond) - cancel() - }() - result, err := myLimitedFunction(ctx, 2) - t.Log(result, err) // Output: 0 context canceled - result, _ = myLimitedFunction(context.Background(), 3) - t.Log(result) // Output: 4 -}