diff --git a/query.go b/query.go
index c1c6457..463c0d2 100644
--- a/query.go
+++ b/query.go
@@ -10,8 +10,10 @@ import (
"fmt"
"io"
"net/http"
+ nurl "net/url"
"os"
"strings"
+ "time"
"github.com/antchfx/xpath"
"golang.org/x/net/html"
@@ -91,19 +93,42 @@ func QuerySelectorAll(top *html.Node, selector *xpath.Expr) []*html.Node {
}
// LoadURL loads the HTML document from the specified URL. Default enabling gzip on a HTTP request.
-func LoadURL(url string) (*html.Node, error) {
+func LoadURL(params ...interface{}) (*html.Node, error) {
+ var (
+ url string
+ proxy *nurl.URL
+ timeout = 10 * time.Second
+ transport = &http.Transport{}
+ )
+ for _, param := range params {
+ switch v := param.(type) {
+ case string:
+ url = v
+ case *nurl.URL:
+ proxy = v
+ case time.Duration:
+ timeout = v
+ }
+
+ }
+ if proxy != nil {
+ transport.Proxy = http.ProxyURL(proxy)
+ }
+ client := &http.Client{
+ Transport: transport,
+ Timeout: timeout,
+ }
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
// Enable gzip compression.
req.Header.Add("Accept-Encoding", "gzip")
- resp, err := http.DefaultClient.Do(req)
+ resp, err := client.Do(req)
if err != nil {
return nil, err
}
var reader io.ReadCloser
-
defer func() {
if reader != nil {
reader.Close()
diff --git a/query_test.go b/query_test.go
index 5d92a2f..5b3c1a7 100644
--- a/query_test.go
+++ b/query_test.go
@@ -3,16 +3,17 @@ package htmlquery
import (
"compress/gzip"
"fmt"
+ "github.com/antchfx/xpath"
+ "golang.org/x/net/html"
"io/ioutil"
"net/http"
"net/http/httptest"
+ "net/url"
"os"
"strings"
"sync"
"testing"
-
- "github.com/antchfx/xpath"
- "golang.org/x/net/html"
+ "time"
)
const htmlSample = `<!DOCTYPE html><html lang="en-US">
@@ -198,3 +199,25 @@ func loadHTML(str string) *html.Node {
}
return node
}
+
+func TestLoadURL_Proxy(t *testing.T) {
+ proxy, err := url.Parse("https://116.106.1.171:4002")
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := LoadURL("https://ip.smartproxy.com/json", proxy, 30*time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ output := OutputHTML(doc, true)
+ t.Log(output)
+}
+
+func TestLoadURL_Timeout(t *testing.T) {
+ node, err := LoadURL("https://ip.smartproxy.com/json", 5*time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ output := OutputHTML(node, true)
+ t.Log(output)
+}