Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improve logic on save file dmsgcurl #242

Merged
54 changes: 21 additions & 33 deletions cmd/dmsgcurl/commands/dmsgcurl.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"log"
"net/http"
"net/url"
Expand Down Expand Up @@ -41,17 +42,17 @@ var (
dmsgcurlTries int
dmsgcurlWait int
dmsgcurlOutput string
stdout bool
replace bool
)

func init() {
RootCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "c", "", "dmsg discovery url default:\n"+skyenv.DmsgDiscAddr)
RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", 1, "number of dmsg servers to connect to")
RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m")
RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "fatal", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m")
RootCmd.Flags().StringVarP(&dmsgcurlData, "data", "d", "", "dmsghttp POST data")
// RootCmd.Flags().StringVarP(&dmsgcurlHeader, "header", "H", "", "Pass custom header(s) to server")
RootCmd.Flags().StringVarP(&dmsgcurlOutput, "out", "o", ".", "output filepath")
RootCmd.Flags().BoolVarP(&stdout, "stdout", "n", false, "output to STDOUT")
RootCmd.Flags().StringVarP(&dmsgcurlOutput, "out", "o", "", "output filepath")
RootCmd.Flags().BoolVarP(&replace, "replace", "r", false, "replace exist file with new downloaded")
RootCmd.Flags().IntVarP(&dmsgcurlTries, "try", "t", 1, "download attempts (0 unlimits)")
RootCmd.Flags().IntVarP(&dmsgcurlWait, "wait", "w", 0, "time to wait between fetches")
RootCmd.Flags().StringVarP(&dmsgcurlAgent, "agent", "a", "dmsgcurl/"+buildinfo.Version(), "identify as `AGENT`")
Expand Down Expand Up @@ -137,8 +138,10 @@ var RootCmd = &cobra.Command{
}
fmt.Println(string(respBody))
} else {

file, err := parseOutputFile(dmsgcurlOutput, u.URL.Path)
file := os.Stdout
if dmsgcurlOutput != "" {
file, err = parseOutputFile(dmsgcurlOutput, replace)
}
if err != nil {
return fmt.Errorf("failed to prepare output file: %w", err)
}
Expand All @@ -162,23 +165,11 @@ var RootCmd = &cobra.Command{
httpC := http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgC)}

for i := 0; i < dmsgcurlTries; i++ {
if !stdout {
if dmsgcurlOutput != "" {
dmsgcurlLog.Debugf("Download attempt %d/%d ...", i, dmsgcurlTries)
}

if _, err := file.Seek(0, 0); err != nil {
return fmt.Errorf("failed to reset file: %w", err)
}
if stdout {
if fErr := file.Close(); fErr != nil {
dmsgcurlLog.WithError(fErr).Warn("Failed to close output file.")
}
if err != nil {
if rErr := os.RemoveAll(file.Name()); rErr != nil {
dmsgcurlLog.WithError(rErr).Warn("Failed to remove output file.")
}
if _, err := file.Seek(0, 0); err != nil {
return fmt.Errorf("failed to reset file: %w", err)
}
file = os.Stdout
}
if err := Download(ctx, dmsgcurlLog, &httpC, file, u.URL.String(), 0); err != nil {
dmsgcurlLog.WithError(err).Error()
Expand Down Expand Up @@ -243,27 +234,24 @@ func parseURL(args []string) (*URL, error) {
return &out, nil
}

func parseOutputFile(name string, urlPath string) (*os.File, error) {
stat, statErr := os.Stat(name)
func parseOutputFile(output string, replace bool) (*os.File, error) {
_, statErr := os.Stat(output)
if statErr != nil {
if os.IsNotExist(statErr) {
f, err := os.Create(name) //nolint
if err := os.MkdirAll(filepath.Dir(output), fs.ModePerm); err != nil {
return nil, err
}
f, err := os.Create(output) //nolint
if err != nil {
return nil, err
}
return f, nil
}
return nil, statErr
}

if stat.IsDir() {
f, err := os.Create(filepath.Join(name, urlPath)) //nolint
if err != nil {
return nil, err
}
return f, nil
if replace {
return os.OpenFile(filepath.Clean(output), os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
}

return nil, os.ErrExist
}

Expand Down Expand Up @@ -357,7 +345,7 @@ func (pw *ProgressWriter) Write(p []byte) (int, error) {
current := atomic.AddInt64(&pw.Current, int64(n))
total := atomic.LoadInt64(&pw.Total)
pc := fmt.Sprintf("%d%%", current*100/total)
if !stdout {
if dmsgcurlOutput != "" {
fmt.Printf("Downloading: %d/%dB (%s)", current, total, pc)
if current != total {
fmt.Print("\r")
Expand Down
26 changes: 16 additions & 10 deletions docs/dmsgcurl.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ $ dmsgcurl --help
-c, --dmsg-disc string dmsg discovery url default:
http://dmsgd.skywire.skycoin.com
-l, --loglvl string [ debug | warn | error | fatal | panic | trace | info ]
-o, --out string output filepath (default ".")
-o, --out string output filepath
-r, --replace bool if be true then downloaded exist file will replace by new downloaded
-e, --sess int number of dmsg servers to connect to (default 1)
-s, --sk cipher.SecKey a random key is generated if unspecified
(default 0000000000000000000000000000000000000000000000000000000000000000)
-n, --stdout output to STDOUT
-t, --try int download attempts (0 unlimits) (default 1)
-v, --version version for dmsgcurl
-w, --wait int time to wait between fetches
Expand All @@ -35,34 +35,40 @@ First, lets create a folder where we will host files to serve over `dmsg` and cr

```shell script
// Create serving folder.
$ mkdir /tmp/dmsghttp -p
$ mkdir /tmp/dmsghttp/inner -p

// Create file.
// Create files.
$ echo 'Hello World!' > /tmp/dmsghttp/hello.txt
$ echo 'Hello World!, Inner!' > /tmp/dmsghttp/inner/inner-hello.txt
```

Next, let's serve this over `http` via `dmsg` as transport. We have an example exec for this located within `/example/dmsgget/dmsg-example-http-server`.
Next, let's serve this over `http` via `dmsg` as transport. We have an example exec for this located within `/example/dmsgcurl/dmsg-example-http-server`.

```shell script
# Generate public/private key pair
$ go run ./examples/dmsgget/gen-keys/gen-keys.go
$ go run ./examples/dmsgcurl/gen-keys/gen-keys.go
# PK: 038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851
# SK: e5740e093bd472c2730b0a58944a5dee220d415de62acf45d1c559f56eea2b2d

# Run dmsg http server.
# (replace 'e5740e093bd472c2730b0a58944a5dee220d415de62acf45d1c559f56eea2b2d' with the SK returned from above command)
$ go run ./examples/dmsgget/dmsg-example-http-server/dmsg-example-http-server.go --dir /tmp/dmsghttp --sk e5740e093bd472c2730b0a58944a5dee220d415de62acf45d1c559f56eea2b2d
$ go run ./examples/dmsgcurl/dmsg-example-http-server/dmsg-example-http-server.go --dir /tmp/dmsghttp --sk e5740e093bd472c2730b0a58944a5dee220d415de62acf45d1c559f56eea2b2d
```

Now we can use `dmsgcurl` to download the hosted file. Open a new terminal and run the following.

```shell script
# Replace '038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851' with the generated PK.
$ dmsgcurl dmsg://038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851:80/hello.txt
$ dmsgcurl dmsg://038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851:80/hello.txt # Output be here as stdout
$ dmsgcurl dmsg://038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851:80/hello.txt -o downloadedFile/hello.txt
$ dmsgcurl dmsg://038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851:80/inner/inner-hello.txt # Output be here as stdout
$ dmsgcurl dmsg://038dde2d050803db59e2ad19e5a6db0f58f8419709fc65041c48b0cb209bb7a851:80/inner/inner-hello.txt -o inner-hello.txt

# Check downloaded file.
$ cat hello.txt
# Hello World!
$ cat downloadedFile/hello.txt
# Hello World!, Inner!
$ cat inner-hello.txt
# Hello World!, Inner!
```

Note: If you set `-d` or `--data` flag, then curl work as post method (upload), and if not then work as get method (download).
Loading