Merge pull request #392 from rosahaj/master

Add client.SetMultipartBoundaryFunc and port Blink/WebKit/Firefox implementations

Author: imroc

client.go

@@ -59,6 +59,7 @@ type Client struct {
 	jsonUnmarshal           func(data []byte, v interface{}) error
 	xmlMarshal              func(v interface{}) ([]byte, error)
 	xmlUnmarshal            func(data []byte, v interface{}) error
+	multipartBoundaryFunc   func() string
 	outputDirectory         string
 	scheme                  string
 	log                     Logger
@@ -239,6 +240,17 @@ func (c *Client) SetCommonFormData(data map[string]string) *Client {
 	return c
 }
 
+// SetMultipartBoundaryFunc overrides the default function used to generate
+// boundary delimiters for "multipart/form-data" requests with a customized one,
+// which returns a boundary delimiter (without the two leading hyphens).
+//
+// Boundary delimiter may only contain certain ASCII characters, and must be
+// non-empty and at most 70 bytes long (see RFC 2046, Section 5.1.1).
+func (c *Client) SetMultipartBoundaryFunc(fn func() string) *Client {
+	c.multipartBoundaryFunc = fn
+	return c
+}
+
 // SetBaseURL set the default base URL, will be used if request URL is
 // a relative URL.
 func (c *Client) SetBaseURL(u string) *Client {

client_impersonate.go

@@ -1,10 +1,56 @@
 package req
 
 import (
+	"crypto/rand"
+	"encoding/binary"
+	"math/big"
+	"strconv"
+	"strings"
+
 	"github.com/imroc/req/v3/http2"
 	utls "github.com/refraction-networking/utls"
 )
 
+// Identical for both Blink-based browsers (Chrome, Chromium, etc.) and WebKit-based browsers (Safari, etc.)
+// Blink implementation: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/network/form_data_encoder.cc;drc=1d694679493c7b2f7b9df00e967b4f8699321093;l=130
+// WebKit implementation: https://github.com/WebKit/WebKit/blob/47eea119fe9462721e5cc75527a4280c6d5f5214/Source/WebCore/platform/network/FormDataBuilder.cpp#L120
+func webkitMultipartBoundaryFunc() string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789AB"
+
+	sb := strings.Builder{}
+	sb.WriteString("----WebKitFormBoundary")
+
+	for i := 0; i < 16; i++ {
+		index, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)-1)))
+		if err != nil {
+			panic(err)
+		}
+
+		sb.WriteByte(letters[index.Int64()])
+	}
+
+	return sb.String()
+}
+
+// Firefox implementation: https://searchfox.org/mozilla-central/source/dom/html/HTMLFormSubmission.cpp#355
+func firefoxMultipartBoundaryFunc() string {
+	sb := strings.Builder{}
+	sb.WriteString("-------------------------")
+
+	for i := 0; i < 3; i++ {
+		var b [8]byte
+		if _, err := rand.Read(b[:]); err != nil {
+			panic(err)
+		}
+		u32 := binary.LittleEndian.Uint32(b[:])
+		s := strconv.FormatUint(uint64(u32), 10)
+
+		sb.WriteString(s)
+	}
+
+	return sb.String()
+}
+
 var (
 	chromeHttp2Settings = []http2.Setting{
 		{
@@ -71,6 +117,7 @@ var (
 		"sec-fetch-dest":            "document",
 		"accept-language":           "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,it;q=0.6",
 	}
+
 	chromeHeaderPriority = http2.PriorityParam{
 		StreamDep: 0,
 		Exclusive: true,
@@ -87,7 +134,8 @@ func (c *Client) ImpersonateChrome() *Client {
 		SetCommonPseudoHeaderOder(chromePseudoHeaderOrder...).
 		SetCommonHeaderOrder(chromeHeaderOrder...).
 		SetCommonHeaders(chromeHeaders).
-		SetHTTP2HeaderPriority(chromeHeaderPriority)
+		SetHTTP2HeaderPriority(chromeHeaderPriority).
+		SetMultipartBoundaryFunc(webkitMultipartBoundaryFunc)
 	return c
 }
 
@@ -106,6 +154,7 @@ var (
 			Val: 16384,
 		},
 	}
+
 	firefoxPriorityFrames = []http2.PriorityFrame{
 		{
 			StreamID: 3,
@@ -156,12 +205,14 @@ var (
 			},
 		},
 	}
+
 	firefoxPseudoHeaderOrder = []string{
 		":method",
 		":path",
 		":authority",
 		":scheme",
 	}
+
 	firefoxHeaderOrder = []string{
 		"user-agent",
 		"accept",
@@ -176,6 +227,7 @@ var (
 		"sec-fetch-user",
 		"te",
 	}
+
 	firefoxHeaders = map[string]string{
 		"user-agent":                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:105.0) Gecko/20100101 Firefox/105.0",
 		"accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -187,6 +239,7 @@ var (
 		"sec-fetch-user":            "?1",
 		//"te":                        "trailers",
 	}
+
 	firefoxHeaderPriority = http2.PriorityParam{
 		StreamDep: 13,
 		Exclusive: false,
@@ -204,7 +257,8 @@ func (c *Client) ImpersonateFirefox() *Client {
 		SetCommonPseudoHeaderOder(firefoxPseudoHeaderOrder...).
 		SetCommonHeaderOrder(firefoxHeaderOrder...).
 		SetCommonHeaders(firefoxHeaders).
-		SetHTTP2HeaderPriority(firefoxHeaderPriority)
+		SetHTTP2HeaderPriority(firefoxHeaderPriority).
+		SetMultipartBoundaryFunc(firefoxMultipartBoundaryFunc)
 	return c
 }
 
@@ -264,6 +318,7 @@ func (c *Client) ImpersonateSafari() *Client {
 		SetCommonPseudoHeaderOder(safariPseudoHeaderOrder...).
 		SetCommonHeaderOrder(safariHeaderOrder...).
 		SetCommonHeaders(safariHeaders).
-		SetHTTP2HeaderPriority(safariHeaderPriority)
+		SetHTTP2HeaderPriority(safariHeaderPriority).
+		SetMultipartBoundaryFunc(webkitMultipartBoundaryFunc)
 	return c
 }

client_test.go

@@ -5,12 +5,14 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -468,6 +470,35 @@ func TestSetCommonFormData(t *testing.T) {
 	tests.AssertEqual(t, "test", form.Get("test"))
 }
 
+func TestSetMultipartBoundaryFunc(t *testing.T) {
+	delimiter := "test-delimiter"
+	expectedContentType := fmt.Sprintf("multipart/form-data; boundary=%s", delimiter)
+	resp, err := tc().
+		SetMultipartBoundaryFunc(func() string {
+			return delimiter
+		}).R().
+		EnableForceMultipart().
+		SetFormData(
+			map[string]string{
+				"test": "test",
+			}).
+		Post("/content-type")
+	assertSuccess(t, resp, err)
+	tests.AssertEqual(t, expectedContentType, resp.String())
+}
+
+func TestFirefoxMultipartBoundaryFunc(t *testing.T) {
+	r := regexp.MustCompile(`^-------------------------\d{1,10}\d{1,10}\d{1,10}$`)
+	b := firefoxMultipartBoundaryFunc()
+	tests.AssertEqual(t, true, r.MatchString(b))
+}
+
+func TestWebkitMultipartBoundaryFunc(t *testing.T) {
+	r := regexp.MustCompile(`^----WebKitFormBoundary[0-9a-zA-Z]{16}$`)
+	b := webkitMultipartBoundaryFunc()
+	tests.AssertEqual(t, true, r.MatchString(b))
+}
+
 func TestClientClone(t *testing.T) {
 	c1 := tc().DevMode().
 		SetCommonHeader("test", "test").

client_wrapper.go

@@ -3,13 +3,14 @@ package req
 import (
 	"context"
 	"crypto/tls"
-	"github.com/imroc/req/v3/http2"
-	utls "github.com/refraction-networking/utls"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"time"
+
+	"github.com/imroc/req/v3/http2"
+	utls "github.com/refraction-networking/utls"
 )
 
 // WrapRoundTrip is a global wrapper methods which delegated
@@ -56,6 +57,12 @@ func SetCommonFormData(data map[string]string) *Client {
 	return defaultClient.SetCommonFormData(data)
 }
 
+// SetMultipartBoundaryFunc is a global wrapper methods which delegated
+// to the default client's Client.SetMultipartBoundaryFunc.
+func SetMultipartBoundaryFunc(fn func() string) *Client {
+	return defaultClient.SetMultipartBoundaryFunc(fn)
+}
+
 // SetBaseURL is a global wrapper methods which delegated
 // to the default client's Client.SetBaseURL.
 func SetBaseURL(u string) *Client {
@@ -482,6 +489,18 @@ func ImpersonateChrome() *Client {
 	return defaultClient.ImpersonateChrome()
 }
 
+// ImpersonateChrome is a global wrapper methods which delegated
+// to the default client's Client.ImpersonateChrome.
+func ImpersonateFirefox() *Client {
+	return defaultClient.ImpersonateFirefox()
+}
+
+// ImpersonateChrome is a global wrapper methods which delegated
+// to the default client's Client.ImpersonateChrome.
+func ImpersonateSafari() *Client {
+	return defaultClient.ImpersonateFirefox()
+}
+
 // SetCommonContentType is a global wrapper methods which delegated
 // to the default client's Client.SetCommonContentType.
 func SetCommonContentType(ct string) *Client {

middleware.go

@@ -148,13 +148,21 @@ func writeMultiPart(r *Request, w *multipart.Writer) {
 	}
 }
 
-func handleMultiPart(r *Request) (err error) {
+func handleMultiPart(c *Client, r *Request) (err error) {
+	var b string
+	if c.multipartBoundaryFunc != nil {
+		b = c.multipartBoundaryFunc()
+	}
+
 	if r.forceChunkedEncoding {
 		pr, pw := io.Pipe()
 		r.GetBody = func() (io.ReadCloser, error) {
 			return pr, nil
 		}
 		w := multipart.NewWriter(pw)
+		if len(b) > 0 {
+			w.SetBoundary(b)
+		}
 		r.SetContentType(w.FormDataContentType())
 		go func() {
 			writeMultiPart(r, w)
@@ -163,6 +171,9 @@ func handleMultiPart(r *Request) (err error) {
 	} else {
 		buf := new(bytes.Buffer)
 		w := multipart.NewWriter(buf)
+		if len(b) > 0 {
+			w.SetBoundary(b)
+		}
 		writeMultiPart(r, w)
 		r.GetBody = func() (io.ReadCloser, error) {
 			return io.NopCloser(bytes.NewReader(buf.Bytes())), nil
@@ -242,7 +253,7 @@ func parseRequestBody(c *Client, r *Request) (err error) {
 	}
 	// handle multipart
 	if r.isMultiPart {
-		return handleMultiPart(r)
+		return handleMultiPart(c, r)
 	}
 
 	// handle form data

response.go

@@ -1,12 +1,13 @@
 package req
 
 import (
-	"github.com/imroc/req/v3/internal/header"
-	"github.com/imroc/req/v3/internal/util"
 	"io"
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/imroc/req/v3/internal/header"
+	"github.com/imroc/req/v3/internal/util"
 )
 
 // Response is the http response.