From 71fa9e67438408808216d23d6230cba389e11aa3 Mon Sep 17 00:00:00 2001 From: blacknon Date: Thu, 28 Dec 2023 22:20:41 +0900 Subject: [PATCH 1/5] =?UTF-8?q?update.=20proxy=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/client.go | 6 +++++- client/gitlab.go | 43 ++++++++++++++++++++++--------------------- cmd/copy.go | 5 +++++ cmd/init.go | 2 +- cmd/root.go | 4 +++- 5 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 cmd/copy.go diff --git a/client/client.go b/client/client.go index 0389bf3..a19f55f 100644 --- a/client/client.go +++ b/client/client.go @@ -31,7 +31,11 @@ func (c *Client) Init(conf config.Config) { // Gitlab.Init for _, gitlabConf := range conf.GitLab { - g := GitlabClient{} + g := GitlabClient{ + proxy: gitlabConf.Proxy, + proxyUser: gitlabConf.ProxyUser, + proxyPass: gitlabConf.ProxyPass, + } g.Init(gitlabConf.Url, gitlabConf.AccessToken) c.lists = append(c.lists, &g) diff --git a/client/gitlab.go b/client/gitlab.go index 6c134d4..5426d45 100644 --- a/client/gitlab.go +++ b/client/gitlab.go @@ -6,6 +6,7 @@ package client import ( "context" + "encoding/base64" "fmt" "net/http" "net/url" @@ -22,6 +23,11 @@ type GitlabClient struct { PlatformName string FilterKey string Project *gitlab.Project + + // proxy + proxy string + proxyUser string + proxyPass string } var ( @@ -35,27 +41,22 @@ func (g *GitlabClient) Init(u, token string) (err error) { // create ctx g.ctx = context.Background() - // TODO: proxyを設定できるように修正する(↓はテスト時のコードなので少し残しておく) - // Create http client - // h := &http.Client{ - // Transport: &http.Transport{ - // DialContext: (&net.Dialer{ - // Timeout: 1000 * time.Millisecond, - // KeepAlive: 1000 * time.Millisecond, - // }).DialContext, - // TLSHandshakeTimeout: 300 * time.Millisecond, - // ResponseHeaderTimeout: 300 * time.Millisecond, - // ExpectContinueTimeout: 100 * time.Millisecond, - // }, - // Timeout: 3000 * time.Millisecond, - // } - // proxyUrl, err := url.Parse("http://127.0.0.1:8080") - // h := &http.Client{ - // Transport: &http.Transport{ - // Proxy: http.ProxyURL(proxyUrl), - // }, - // } - h := &http.Client{} + transport := &http.Transport{} + if g.proxy != "" { + proxyUrl, err := url.Parse(g.proxy) + if err != nil { + return err + } + + hdr := make(http.Header) + hdr.Add("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(g.proxyUser+":"+g.proxyPass))) + + transport = &http.Transport{ + Proxy: http.ProxyURL(proxyUrl), + } + } + + h := &http.Client{Transport: transport} // Create Gitlab Client g.client, err = gitlab.NewClient(token, gitlab.WithBaseURL(u), gitlab.WithHTTPClient(h)) diff --git a/cmd/copy.go b/cmd/copy.go new file mode 100644 index 0000000..58973b2 --- /dev/null +++ b/cmd/copy.go @@ -0,0 +1,5 @@ +// Copyright (c) 2023 Blacknon. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. + +package cmd diff --git a/cmd/init.go b/cmd/init.go index 05722ac..6f92978 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -12,7 +12,7 @@ import ( ) var ( - // ssss + // config file configFileName = "config.toml" ) diff --git a/cmd/root.go b/cmd/root.go index 2f72b26..88b6a9c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( var App = &cli.App{ Name: "snipt", Usage: "multiple remote platform snippet manager.", - Version: "0.1.3", + Version: "0.1.4", ErrWriter: ioutil.Discard, // Flags @@ -45,6 +45,8 @@ var App = &cli.App{ // add subcommand // comment subcommand + + // copy subcommand }, // Output usages and error messages From 4376766491cd53cd205caa0143f624f490ac0bbb Mon Sep 17 00:00:00 2001 From: blacknon Date: Sun, 14 Apr 2024 00:24:22 +0900 Subject: [PATCH 2/5] =?UTF-8?q?update.=20goroutine=E3=81=A7platform?= =?UTF-8?q?=E3=82=92=E3=83=91=E3=83=A9=E3=83=AC=E3=83=AB=E5=8F=96=E5=BE=97?= =?UTF-8?q?=E3=81=95=E3=81=9B=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/client.go | 115 +++++++++++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/client/client.go b/client/client.go index a19f55f..534e1a6 100644 --- a/client/client.go +++ b/client/client.go @@ -7,6 +7,7 @@ package client import ( "fmt" "os" + "sync" "github.com/blacknon/snipt/config" "github.com/google/go-github/github" @@ -140,55 +141,93 @@ func (c *Client) Delete(url string) (err error) { } // PlatformList -func (c *Client) PlatformList(enableProject bool) (platformList []string, err error) { +func (c *Client) PlatformList(enableProject bool) ([]string, error) { + var platformList []string + var err error + var wg sync.WaitGroup // 同期用のWaitGroupを用意 + // clear c.filterListsData = []*SnippetListData{} - for _, gc := range c.lists { - platformName := gc.GetPlatformName() - gc.SetFilterKey(platformName) - - // append platform to platformList - platformList = append(platformList, platformName) + // チャネルを用意して、処理結果を収集 + resultChannel := make(chan struct { + platform string + data *SnippetListData + err error + }, len(c.lists)) - // append paltform to c.filterListsData - data := &SnippetListData{ - Client: gc, - Platform: platformName, - } + // 各リストに対して並行処理を実行 + for _, gc := range c.lists { + wg.Add(1) + go func(gc GitClient) { + defer wg.Done() - c.filterListsData = append(c.filterListsData, data) + platformName := gc.GetPlatformName() + gc.SetFilterKey(platformName) - // Get gitlab project list - glsnippet, ok := gc.(*GitlabClient) - if enableProject && ok { - projects, err := glsnippet.GetProjectList() - if err != nil { - return []string{}, err + data := &SnippetListData{ + Client: gc, + Platform: platformName, } - for _, p := range projects { - // set platformName - pn := fmt.Sprintf("%s /%s", platformName, p.PathWithNamespace) - - // set pd - pd := &SnippetListData{} - *pd = *data - pd.Platform = platformName - - // set pd.client - gls := &GitlabClient{} - *gls = *glsnippet - gls.Project = p - gls.SetFilterKey(pn) - pd.Client = gls - - platformList = append(platformList, pn) - c.filterListsData = append(c.filterListsData, pd) + // Get gitlab project list + glsnippet, ok := gc.(*GitlabClient) + if enableProject && ok { + projects, err := glsnippet.GetProjectList() + if err != nil { + resultChannel <- struct { + platform string + data *SnippetListData + err error + }{platform: platformName, data: nil, err: err} + return + } + + for _, p := range projects { + pn := fmt.Sprintf("%s /%s", platformName, p.PathWithNamespace) + + pd := &SnippetListData{ + Client: &GitlabClient{Project: p /* 他のフィールドを設定 */}, + Platform: pn, + } + // 結果をチャネルに送信 + resultChannel <- struct { + platform string + data *SnippetListData + err error + }{platform: pn, data: pd, err: nil} + } + } else { + // 結果をチャネルに送信 + resultChannel <- struct { + platform string + data *SnippetListData + err error + }{platform: platformName, data: data, err: nil} } + }(gc) + } + + // 全てのgoroutineが終了するのを待機 + go func() { + wg.Wait() + close(resultChannel) // チャネルを閉じる + }() + + // 結果を受け取ってリストに追加 + for result := range resultChannel { + if result.err != nil { + err = result.err + } else { + platformList = append(platformList, result.platform) + c.filterListsData = append(c.filterListsData, result.data) } } - return + + if err != nil { + return nil, err + } + return platformList, nil } func (c *Client) VisibilityListFromPlatform(platform string) (visibilityList []Visibility) { From cc3b80bafa705499b801e46987e0a113c3c4645e Mon Sep 17 00:00:00 2001 From: blacknon Date: Sun, 14 Apr 2024 00:41:15 +0900 Subject: [PATCH 3/5] =?UTF-8?q?update.=20goroutine=E3=81=A7platform?= =?UTF-8?q?=E3=82=92=E3=83=91=E3=83=A9=E3=83=AC=E3=83=AB=E5=8F=96=E5=BE=97?= =?UTF-8?q?=E3=81=95=E3=81=9B=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/client.go | 39 ++++++++++++++++++++++++++++++--------- client/gist.go | 6 ++++++ cmd/root.go | 8 +++----- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/client/client.go b/client/client.go index 534e1a6..55113f7 100644 --- a/client/client.go +++ b/client/client.go @@ -44,22 +44,43 @@ func (c *Client) Init(conf config.Config) { } // List -func (c *Client) List(isFile, isSecret bool) (snippetList SnippetList) { - // +func (c *Client) List(isFile, isSecret bool) SnippetList { + var snippetList SnippetList + var wg sync.WaitGroup // goroutineの完了を待機するためのWaitGroup + resultChannel := make(chan SnippetList, len(c.lists)) // 処理結果を収集するチャネル + + // 各クライアントに対してgoroutineを起動 for _, gc := range c.lists { - list, err := gc.List(isFile, isSecret) - if err != nil { - fmt.Fprintf(os.Stderr, "Error: %s\n", err) - continue - } + wg.Add(1) + go func(gc GitClient) { + defer wg.Done() + // クライアントのListメソッドを実行 + list, err := gc.List(isFile, isSecret) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + return + } + // 処理結果をチャネルに送信 + resultChannel <- list + }(gc) + } + + // 全てのgoroutineの完了を待機してチャネルを閉じる + go func() { + wg.Wait() + close(resultChannel) + }() + + // チャネルから受け取ったリストをまとめる + for list := range resultChannel { snippetList = append(snippetList, list...) } - // + // filterListsDataに結果を保存 c.filterListsData = snippetList - return + return snippetList } // Get diff --git a/client/gist.go b/client/gist.go index 65465c4..8399824 100644 --- a/client/gist.go +++ b/client/gist.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "net/url" + "sort" "github.com/google/go-github/github" "golang.org/x/oauth2" @@ -98,6 +99,11 @@ func (g *GistClient) List(isFile, isSecret bool) (snippetList SnippetList, err e } } + sort.Slice(snippetList, func(i, j int) bool { + // i番目とj番目の要素のAgeを比較 + return snippetList[i].URL < snippetList[j].URL + }) + return snippetList, err } diff --git a/cmd/root.go b/cmd/root.go index 88b6a9c..489f244 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" - "io/ioutil" "os" "github.com/urfave/cli/v2" @@ -14,10 +13,9 @@ import ( // App var App = &cli.App{ - Name: "snipt", - Usage: "multiple remote platform snippet manager.", - Version: "0.1.4", - ErrWriter: ioutil.Discard, + Name: "snipt", + Usage: "multiple remote platform snippet manager.", + Version: "0.1.4", // Flags Flags: commonFlags, From 3b47306755f008ac9983dc65474fff22658e5e3a Mon Sep 17 00:00:00 2001 From: blacknon Date: Sun, 14 Apr 2024 12:39:26 +0900 Subject: [PATCH 4/5] update. bugfix #6. --- client/gitlab.go | 90 ++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/client/gitlab.go b/client/gitlab.go index 5426d45..fdb3493 100644 --- a/client/gitlab.go +++ b/client/gitlab.go @@ -82,50 +82,72 @@ func (g *GitlabClient) Init(u, token string) (err error) { // List func (g *GitlabClient) List(isFile, isSecret bool) (snippetList SnippetList, err error) { - // get snippetList - snippetDataList, _, err := g.client.Snippets.ListSnippets(&gitlab.ListSnippetsOptions{}) - - // gitlab.Snippet to Interface - for _, snippet := range snippetDataList { - if !isSecret { - // - v := getGitlabVisibilityFromString(snippet.Visibility) - switch v { - case GitlabIsPrivate, GitlabIsInternal: - continue - } - } + // set ListProjectsOptions pageSize + const pageSize = 50 - // get Description - title := replaceNewline(snippet.Title, "\\n") + opt := gitlab.ListSnippetsOptions( + gitlab.ListOptions{ + Page: 0, + PerPage: pageSize, + }, + ) - data := SnippetListData{ - Client: g, - Platform: g.PlatformName, - Id: strconv.Itoa(snippet.ID), - Title: title, - URL: snippet.WebURL, - Visibility: snippet.Visibility, + for { + // get snippetList + snippetDataList, resp, ferr := g.client.Snippets.ListSnippets(&opt) + + // check error + if ferr != nil { + return snippetList, ferr } - // check file flag - if isFile { - if len(snippet.Files) > 1 { - for _, f := range snippet.Files { - fd := data - fd.URL, _ = url.JoinPath(fd.URL, f.Path) - fd.RawURL = f.RawURL + for _, snippet := range snippetDataList { + if !isSecret { + // + v := getGitlabVisibilityFromString(snippet.Visibility) + switch v { + case GitlabIsPrivate, GitlabIsInternal: + continue + } + } + + // get Description + title := replaceNewline(snippet.Title, "\\n") - snippetList = append(snippetList, &fd) + data := SnippetListData{ + Client: g, + Platform: g.PlatformName, + Id: strconv.Itoa(snippet.ID), + Title: title, + URL: snippet.WebURL, + Visibility: snippet.Visibility, + } + + // check file flag + if isFile { + if len(snippet.Files) > 1 { + for _, f := range snippet.Files { + fd := data + fd.URL, _ = url.JoinPath(fd.URL, f.Path) + fd.RawURL = f.RawURL + + snippetList = append(snippetList, &fd) + } + } else { + data.URL, _ = url.JoinPath(data.URL, snippet.FileName) + data.RawURL = snippet.RawURL + snippetList = append(snippetList, &data) } } else { - data.URL, _ = url.JoinPath(data.URL, snippet.FileName) - data.RawURL = snippet.RawURL snippetList = append(snippetList, &data) } - } else { - snippetList = append(snippetList, &data) } + + if resp.NextPage == 0 { + break + } + + opt.Page = resp.NextPage } return From d8ead2546775d46103c6e08f9e720050983ae4b6 Mon Sep 17 00:00:00 2001 From: blacknon Date: Sun, 14 Apr 2024 12:43:15 +0900 Subject: [PATCH 5/5] update. --- cmd/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/create.go b/cmd/create.go index 5db8a19..d75dde2 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -118,7 +118,7 @@ func cmdActionCreate(c *cli.Context) (err error) { } rawURL, eErr := cl.Create(t, snippetData) - if err != nil { + if eErr != nil { return eErr }