From f1f1c5e95e08bdc0f544759b1c23b0436dfd45a1 Mon Sep 17 00:00:00 2001 From: xrgzs Date: Sat, 27 Jul 2024 22:52:13 +0800 Subject: [PATCH] Revert "perf(123pan): optimize rate limiting (#6859)" This reverts commit #6859, which introduced performance enhancements in #6859. Unfortunately, these changes led to unintended consequences resulting in account bans. The revert is necessary until a more thorough review and testing can be conducted to ensure no negative impacts on account status. --- drivers/123/driver.go | 16 +++++++++------- drivers/123/util.go | 10 +++++----- drivers/123_share/driver.go | 13 ++++++------- drivers/123_share/util.go | 8 ++++---- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index aeda7fcf742..bd7089e85d2 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -53,7 +53,7 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) + files, err := d.getFiles(dir.GetID(), dir.GetName()) if err != nil { return nil, err } @@ -247,6 +247,9 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } _, err = uploader.UploadWithContext(ctx, input) } + if err != nil { + return err + } _, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": resp.Data.FileId, @@ -255,12 +258,11 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } -func (d *Pan123) APIRateLimit(ctx context.Context, api string) error { - value, _ := d.apiRateLimit.LoadOrStore(api, - rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) - limiter := value.(*rate.Limiter) - - return limiter.Wait(ctx) +func (d *Pan123) APIRateLimit(api string) bool { + limiter, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) + ins := limiter.(*rate.Limiter) + return ins.Allow() } var _ driver.Driver = (*Pan123)(nil) diff --git a/drivers/123/util.go b/drivers/123/util.go index 73c73b3b3b3..540201f7035 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -1,7 +1,6 @@ package _123 import ( - "context" "errors" "fmt" "hash/crc32" @@ -15,7 +14,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/go-resty/resty/v2" + resty "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) @@ -234,14 +233,15 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r return body, nil } -func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { +func (d *Pan123) getFiles(parentId string, name string) ([]File, error) { page := 1 total := 0 res := make([]File, 0) // 2024-02-06 fix concurrency by 123pan for { - if err := d.APIRateLimit(ctx, FileList); err != nil { - return nil, err + if !d.APIRateLimit(FileList) { + time.Sleep(time.Millisecond * 200) + continue } var resp Files query := map[string]string{ diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index 9c1f3803710..7fca7cc145e 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -45,7 +45,7 @@ func (d *Pan123Share) Drop(ctx context.Context) error { func (d *Pan123Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // TODO return the files list, required - files, err := d.getFiles(ctx, dir.GetID()) + files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } @@ -150,12 +150,11 @@ func (d *Pan123Share) Put(ctx context.Context, dstDir model.Obj, stream model.Fi // return nil, errs.NotSupport //} -func (d *Pan123Share) APIRateLimit(ctx context.Context, api string) error { - value, _ := d.apiRateLimit.LoadOrStore(api, - rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) - limiter := value.(*rate.Limiter) - - return limiter.Wait(ctx) +func (d *Pan123Share) APIRateLimit(api string) bool { + limiter, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) + ins := limiter.(*rate.Limiter) + return ins.Allow() } var _ driver.Driver = (*Pan123Share)(nil) diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index 80ea8f0ca46..a192993bf19 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -1,7 +1,6 @@ package _123Share import ( - "context" "errors" "fmt" "hash/crc32" @@ -81,12 +80,13 @@ func (d *Pan123Share) request(url string, method string, callback base.ReqCallba return body, nil } -func (d *Pan123Share) getFiles(ctx context.Context, parentId string) ([]File, error) { +func (d *Pan123Share) getFiles(parentId string) ([]File, error) { page := 1 res := make([]File, 0) for { - if err := d.APIRateLimit(ctx, FileList); err != nil { - return nil, err + if !d.APIRateLimit(FileList) { + time.Sleep(time.Millisecond * 200) + continue } var resp Files query := map[string]string{