diff --git a/docs/netutil.md b/docs/netutil.md index 14e0298..18c1a2f 100644 --- a/docs/netutil.md +++ b/docs/netutil.md @@ -8,8 +8,6 @@ Package netutil contains functions to get net information and send http request. - [https://github.com/duke-git/lancet/blob/main/netutil/net.go](https://github.com/duke-git/lancet/blob/main/netutil/net.go) -- [https://github.com/duke-git/lancet/blob/main/netutil/http_client.go](https://github.com/duke-git/lancet/blob/main/netutil/http_client.go) - - [https://github.com/duke-git/lancet/blob/main/netutil/http.go](https://github.com/duke-git/lancet/blob/main/netutil/http.go)
diff --git a/docs/netutil_zh-CN.md b/docs/netutil_zh-CN.md index 428ca9e..b01987b 100644 --- a/docs/netutil_zh-CN.md +++ b/docs/netutil_zh-CN.md @@ -7,7 +7,6 @@ netutil 网络包支持获取 ip 地址，发送 http 请求。 ## 源码: - [https://github.com/duke-git/lancet/blob/main/netutil/net.go](https://github.com/duke-git/lancet/blob/main/netutil/net.go) -- [https://github.com/duke-git/lancet/blob/main/netutil/http_client.go](https://github.com/duke-git/lancet/blob/main/netutil/http_client.go) - [https://github.com/duke-git/lancet/blob/main/netutil/http.go](https://github.com/duke-git/lancet/blob/main/netutil/http.go) @@ -113,8 +112,8 @@ func main() { fmt.Println(err) } - fmt.Println(encodedUrl) - + fmt.Println(encodedUrl) + // Output: // http://www.lancet.com?a=1&b=%5B2%5D } @@ -175,8 +174,8 @@ import ( func main() { ips := netutil.GetIps() - fmt.Println(ips) - + fmt.Println(ips) + // Output: // [192.168.1.9] } diff --git a/netutil/http.go b/netutil/http.go index 1db8f8c..6276380 100644 --- a/netutil/http.go +++ b/netutil/http.go @@ -13,12 +13,21 @@ package netutil import ( + "bytes" + "crypto/tls" "encoding/json" "errors" "fmt" + "io" "net/http" + "net/url" + "reflect" + "regexp" "sort" "strings" + "time" + + "github.com/duke-git/lancet/v2/slice" ) // HttpGet send get http request. @@ -78,3 +87,220 @@ func ConvertMapToQueryString(param map[string]any) string { } return build.String() } + +// HttpRequest struct is a composed http request +type HttpRequest struct { + RawURL string + Method string + Headers http.Header + QueryParams url.Values + FormData url.Values + Body []byte +} + +// HttpClientConfig contains some configurations for http client +type HttpClientConfig struct { + SSLEnabled bool + TLSConfig *tls.Config + Compressed bool + HandshakeTimeout time.Duration + ResponseTimeout time.Duration + Verbose bool +} + +// defaultHttpClientConfig defalut client config +var defaultHttpClientConfig = &HttpClientConfig{ + Compressed: false, + HandshakeTimeout: 20 * time.Second, + ResponseTimeout: 40 * time.Second, +} + +// HttpClient is used for sending http request +type HttpClient struct { + *http.Client + TLS *tls.Config + Request *http.Request + Config HttpClientConfig +} + +// NewHttpClient make a HttpClient instance +func NewHttpClient() *HttpClient { + client := &HttpClient{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSHandshakeTimeout: defaultHttpClientConfig.HandshakeTimeout, + ResponseHeaderTimeout: defaultHttpClientConfig.ResponseTimeout, + DisableCompression: !defaultHttpClientConfig.Compressed, + }, + }, + Config: *defaultHttpClientConfig, + } + + return client +} + +// NewHttpClientWithConfig make a HttpClient instance with pass config +func NewHttpClientWithConfig(config *HttpClientConfig) *HttpClient { + if config == nil { + config = defaultHttpClientConfig + } + + client := &HttpClient{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSHandshakeTimeout: config.HandshakeTimeout, + ResponseHeaderTimeout: config.ResponseTimeout, + DisableCompression: !config.Compressed, + }, + }, + Config: *config, + } + + if config.SSLEnabled { + client.TLS = config.TLSConfig + } + + return client +} + +// SendRequest send http request. +// Play: https://go.dev/play/p/jUSgynekH7G +func (client *HttpClient) SendRequest(request *HttpRequest) (*http.Response, error) { + err := validateRequest(request) + if err != nil { + return nil, err + } + + rawUrl := request.RawURL + + req, err := http.NewRequest(request.Method, rawUrl, bytes.NewBuffer(request.Body)) + if err != nil { + return nil, err + } + + client.setTLS(rawUrl) + client.setHeader(req, request.Headers) + + err = client.setQueryParam(req, rawUrl, request.QueryParams) + if err != nil { + return nil, err + } + + if request.FormData != nil { + client.setFormData(req, request.FormData) + } + + client.Request = req + + resp, err := client.Client.Do(req) + if err != nil { + return nil, err + } + + return resp, nil +} + +// DecodeResponse decode response into target object. +// Play: https://go.dev/play/p/jUSgynekH7G +func (client *HttpClient) DecodeResponse(resp *http.Response, target any) error { + if resp == nil { + return errors.New("invalid target param") + } + defer resp.Body.Close() + return json.NewDecoder(resp.Body).Decode(target) +} + +// setTLS set http client transport TLSClientConfig +func (client *HttpClient) setTLS(rawUrl string) { + if strings.HasPrefix(rawUrl, "https") { + if transport, ok := client.Client.Transport.(*http.Transport); ok { + transport.TLSClientConfig = client.TLS + } + } +} + +// setHeader set http rquest header +func (client *HttpClient) setHeader(req *http.Request, headers http.Header) { + if headers == nil { + headers = make(http.Header) + } + + if _, ok := headers["Accept"]; !ok { + headers["Accept"] = []string{"*/*"} + } + if _, ok := headers["Accept-Encoding"]; !ok && client.Config.Compressed { + headers["Accept-Encoding"] = []string{"deflate, gzip"} + } + + req.Header = headers +} + +// setQueryParam set http request query string param +func (client *HttpClient) setQueryParam(req *http.Request, reqUrl string, queryParam url.Values) error { + if queryParam != nil { + if !strings.Contains(reqUrl, "?") { + reqUrl = reqUrl + "?" + queryParam.Encode() + } else { + reqUrl = reqUrl + "&" + queryParam.Encode() + } + u, err := url.Parse(reqUrl) + if err != nil { + return err + } + req.URL = u + } + return nil +} + +func (client *HttpClient) setFormData(req *http.Request, values url.Values) { + formData := []byte(values.Encode()) + req.Body = io.NopCloser(bytes.NewReader(formData)) + req.ContentLength = int64(len(formData)) +} + +// validateRequest check if a request has url, and valid method. +func validateRequest(req *HttpRequest) error { + if req.RawURL == "" { + return errors.New("invalid request url") + } + + // common HTTP methods + methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", + "HEAD", "CONNECT", "OPTIONS", "TRACE"} + + if !slice.Contain(methods, strings.ToUpper(req.Method)) { + return errors.New("invalid request method") + } + + return nil +} + +// StructToUrlValues convert struct to url valuse, +// only convert the field which is exported and has `json` tag. +// Play: https://go.dev/play/p/pFqMkM40w9z +func StructToUrlValues(targetStruct any) url.Values { + rv := reflect.ValueOf(targetStruct) + rt := reflect.TypeOf(targetStruct) + + if rt.Kind() == reflect.Ptr { + rt = rt.Elem() + } + if rt.Kind() != reflect.Struct { + panic(fmt.Errorf("data type %T not support, shuld be struct or pointer to struct", targetStruct)) + } + + result := url.Values{} + + fieldNum := rt.NumField() + pattern := `^[A-Z]` + regex := regexp.MustCompile(pattern) + for i := 0; i < fieldNum; i++ { + name := rt.Field(i).Name + tag := rt.Field(i).Tag.Get("json") + if regex.MatchString(name) && tag != "" { + result.Add(tag, fmt.Sprintf("%v", rv.Field(i).Interface())) + } + } + + return result +} diff --git a/netutil/http_client.go b/netutil/http_client.go deleted file mode 100644 index 44a1a3a..0000000 --- a/netutil/http_client.go +++ /dev/null @@ -1,235 +0,0 @@ -package netutil - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "reflect" - "regexp" - "strings" - "time" - - "github.com/duke-git/lancet/v2/slice" -) - -// HttpRequest struct is a composed http request -type HttpRequest struct { - RawURL string - Method string - Headers http.Header - QueryParams url.Values - FormData url.Values - Body []byte -} - -// HttpClientConfig contains some configurations for http client -type HttpClientConfig struct { - SSLEnabled bool - TLSConfig *tls.Config - Compressed bool - HandshakeTimeout time.Duration - ResponseTimeout time.Duration - Verbose bool -} - -// defaultHttpClientConfig defalut client config -var defaultHttpClientConfig = &HttpClientConfig{ - Compressed: false, - HandshakeTimeout: 20 * time.Second, - ResponseTimeout: 40 * time.Second, -} - -// HttpClient is used for sending http request -type HttpClient struct { - *http.Client - TLS *tls.Config - Request *http.Request - Config HttpClientConfig -} - -// NewHttpClient make a HttpClient instance -func NewHttpClient() *HttpClient { - client := &HttpClient{ - Client: &http.Client{ - Transport: &http.Transport{ - TLSHandshakeTimeout: defaultHttpClientConfig.HandshakeTimeout, - ResponseHeaderTimeout: defaultHttpClientConfig.ResponseTimeout, - DisableCompression: !defaultHttpClientConfig.Compressed, - }, - }, - Config: *defaultHttpClientConfig, - } - - return client -} - -// NewHttpClientWithConfig make a HttpClient instance with pass config -func NewHttpClientWithConfig(config *HttpClientConfig) *HttpClient { - if config == nil { - config = defaultHttpClientConfig - } - - client := &HttpClient{ - Client: &http.Client{ - Transport: &http.Transport{ - TLSHandshakeTimeout: config.HandshakeTimeout, - ResponseHeaderTimeout: config.ResponseTimeout, - DisableCompression: !config.Compressed, - }, - }, - Config: *config, - } - - if config.SSLEnabled { - client.TLS = config.TLSConfig - } - - return client -} - -// SendRequest send http request. -// Play: https://go.dev/play/p/jUSgynekH7G -func (client *HttpClient) SendRequest(request *HttpRequest) (*http.Response, error) { - err := validateRequest(request) - if err != nil { - return nil, err - } - - rawUrl := request.RawURL - - req, err := http.NewRequest(request.Method, rawUrl, bytes.NewBuffer(request.Body)) - if err != nil { - return nil, err - } - - client.setTLS(rawUrl) - client.setHeader(req, request.Headers) - - err = client.setQueryParam(req, rawUrl, request.QueryParams) - if err != nil { - return nil, err - } - - if request.FormData != nil { - client.setFormData(req, request.FormData) - } - - client.Request = req - - resp, err := client.Client.Do(req) - if err != nil { - return nil, err - } - - return resp, nil -} - -// DecodeResponse decode response into target object. -// Play: https://go.dev/play/p/jUSgynekH7G -func (client *HttpClient) DecodeResponse(resp *http.Response, target any) error { - if resp == nil { - return errors.New("invalid target param") - } - defer resp.Body.Close() - return json.NewDecoder(resp.Body).Decode(target) -} - -// setTLS set http client transport TLSClientConfig -func (client *HttpClient) setTLS(rawUrl string) { - if strings.HasPrefix(rawUrl, "https") { - if transport, ok := client.Client.Transport.(*http.Transport); ok { - transport.TLSClientConfig = client.TLS - } - } -} - -// setHeader set http rquest header -func (client *HttpClient) setHeader(req *http.Request, headers http.Header) { - if headers == nil { - headers = make(http.Header) - } - - if _, ok := headers["Accept"]; !ok { - headers["Accept"] = []string{"*/*"} - } - if _, ok := headers["Accept-Encoding"]; !ok && client.Config.Compressed { - headers["Accept-Encoding"] = []string{"deflate, gzip"} - } - - req.Header = headers -} - -// setQueryParam set http request query string param -func (client *HttpClient) setQueryParam(req *http.Request, reqUrl string, queryParam url.Values) error { - if queryParam != nil { - if !strings.Contains(reqUrl, "?") { - reqUrl = reqUrl + "?" + queryParam.Encode() - } else { - reqUrl = reqUrl + "&" + queryParam.Encode() - } - u, err := url.Parse(reqUrl) - if err != nil { - return err - } - req.URL = u - } - return nil -} - -func (client *HttpClient) setFormData(req *http.Request, values url.Values) { - formData := []byte(values.Encode()) - req.Body = io.NopCloser(bytes.NewReader(formData)) - req.ContentLength = int64(len(formData)) -} - -// validateRequest check if a request has url, and valid method. -func validateRequest(req *HttpRequest) error { - if req.RawURL == "" { - return errors.New("invalid request url") - } - - // common HTTP methods - methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", - "HEAD", "CONNECT", "OPTIONS", "TRACE"} - - if !slice.Contain(methods, strings.ToUpper(req.Method)) { - return errors.New("invalid request method") - } - - return nil -} - -// StructToUrlValues convert struct to url valuse, -// only convert the field which is exported and has `json` tag. -// Play: https://go.dev/play/p/pFqMkM40w9z -func StructToUrlValues(targetStruct any) url.Values { - rv := reflect.ValueOf(targetStruct) - rt := reflect.TypeOf(targetStruct) - - if rt.Kind() == reflect.Ptr { - rt = rt.Elem() - } - if rt.Kind() != reflect.Struct { - panic(fmt.Errorf("data type %T not support, shuld be struct or pointer to struct", targetStruct)) - } - - result := url.Values{} - - fieldNum := rt.NumField() - pattern := `^[A-Z]` - regex := regexp.MustCompile(pattern) - for i := 0; i < fieldNum; i++ { - name := rt.Field(i).Name - tag := rt.Field(i).Tag.Get("json") - if regex.MatchString(name) && tag != "" { - result.Add(tag, fmt.Sprintf("%v", rv.Field(i).Interface())) - } - } - - return result -} diff --git a/netutil/http_client_test.go b/netutil/http_client_test.go deleted file mode 100644 index 41d9fb8..0000000 --- a/netutil/http_client_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package netutil - -import ( - "io" - "log" - "net/http" - "net/url" - "testing" - - "github.com/duke-git/lancet/v2/internal" -) - -func TestHttpClient_Get(t *testing.T) { - assert := internal.NewAssert(t, "TestHttpClient_Get") - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos/1", - Method: "GET", - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - log.Fatal(err) - } - - type Todo struct { - UserId int `json:"userId"` - Id int `json:"id"` - Title string `json:"title"` - Completed bool `json:"completed"` - } - - var todo Todo - err = httpClient.DecodeResponse(resp, &todo) - if err != nil { - t.Log(err) - } - - assert.Equal(1, todo.Id) -} - -func TestHttpClent_Post(t *testing.T) { - header := http.Header{} - header.Add("Content-Type", "multipart/form-data") - - postData := url.Values{} - postData.Add("userId", "1") - postData.Add("title", "testItem") - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos", - Method: "POST", - Headers: header, - FormData: postData, - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil { - log.Fatal(err) - } - - body, _ := io.ReadAll(resp.Body) - t.Log("response: ", resp.StatusCode, string(body)) -} - -func TestStructToUrlValues(t *testing.T) { - assert := internal.NewAssert(t, "TestStructToUrlValues") - - type TodoQuery struct { - Id int `json:"id"` - UserId int `json:"userId"` - } - todoQuery := TodoQuery{ - Id: 1, - UserId: 1, - } - todoValues := StructToUrlValues(todoQuery) - - assert.Equal("1", todoValues.Get("id")) - assert.Equal("1", todoValues.Get("userId")) - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos", - Method: "GET", - QueryParams: todoValues, - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - log.Fatal(err) - } - - body, _ := io.ReadAll(resp.Body) - t.Log("response: ", string(body)) -} diff --git a/netutil/http_example_test.go b/netutil/http_example_test.go deleted file mode 100644 index 71aeebe..0000000 --- a/netutil/http_example_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package netutil - -import "fmt" - -func ExampleHttpClient_SendRequest() { - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos/1", - Method: "GET", - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - return - } - - type Todo struct { - UserId int `json:"userId"` - Id int `json:"id"` - Title string `json:"title"` - Completed bool `json:"completed"` - } - - var todo Todo - err = httpClient.DecodeResponse(resp, &todo) - if err != nil { - return - } - - fmt.Println(todo.Id) - - // Output: - // 1 -} - -func ExampleHttpClient_DecodeResponse() { - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos/1", - Method: "GET", - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - return - } - - type Todo struct { - UserId int `json:"userId"` - Id int `json:"id"` - Title string `json:"title"` - Completed bool `json:"completed"` - } - - var todo Todo - err = httpClient.DecodeResponse(resp, &todo) - if err != nil { - return - } - - fmt.Println(todo.Id) - - // Output: - // 1 -} - -func ExampleStructToUrlValues() { - type TodoQuery struct { - Id int `json:"id"` - Name string `json:"name"` - } - todoQuery := TodoQuery{ - Id: 1, - Name: "Test", - } - todoValues := StructToUrlValues(todoQuery) - - fmt.Println(todoValues.Get("id")) - fmt.Println(todoValues.Get("name")) - - // Output: - // 1 - // Test -} - -func ExampleConvertMapToQueryString() { - var m = map[string]any{ - "c": 3, - "a": 1, - "b": 2, - } - - qs := ConvertMapToQueryString(m) - - fmt.Println(qs) - - // Output: - // a=1&b=2&c=3 -} diff --git a/netutil/http_test.go b/netutil/http_test.go index 5a48828..d32915c 100644 --- a/netutil/http_test.go +++ b/netutil/http_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io" "log" + "net/http" "net/url" "testing" @@ -156,3 +157,90 @@ func TestParseResponse(t *testing.T) { } t.Log("response: ", toDoResp) } + +func TestHttpClient_Get(t *testing.T) { + assert := internal.NewAssert(t, "TestHttpClient_Get") + + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos/1", + Method: "GET", + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil || resp.StatusCode != 200 { + log.Fatal(err) + } + + type Todo struct { + UserId int `json:"userId"` + Id int `json:"id"` + Title string `json:"title"` + Completed bool `json:"completed"` + } + + var todo Todo + err = httpClient.DecodeResponse(resp, &todo) + if err != nil { + t.Log(err) + } + + assert.Equal(1, todo.Id) +} + +func TestHttpClent_Post(t *testing.T) { + header := http.Header{} + header.Add("Content-Type", "multipart/form-data") + + postData := url.Values{} + postData.Add("userId", "1") + postData.Add("title", "testItem") + + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos", + Method: "POST", + Headers: header, + FormData: postData, + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil { + log.Fatal(err) + } + + body, _ := io.ReadAll(resp.Body) + t.Log("response: ", resp.StatusCode, string(body)) +} + +func TestStructToUrlValues(t *testing.T) { + assert := internal.NewAssert(t, "TestStructToUrlValues") + + type TodoQuery struct { + Id int `json:"id"` + UserId int `json:"userId"` + } + todoQuery := TodoQuery{ + Id: 1, + UserId: 1, + } + todoValues := StructToUrlValues(todoQuery) + + assert.Equal("1", todoValues.Get("id")) + assert.Equal("1", todoValues.Get("userId")) + + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos", + Method: "GET", + QueryParams: todoValues, + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil || resp.StatusCode != 200 { + log.Fatal(err) + } + + body, _ := io.ReadAll(resp.Body) + t.Log("response: ", string(body)) +} diff --git a/netutil/net_example_test.go b/netutil/net_example_test.go index a9d967c..c9d7ee6 100644 --- a/netutil/net_example_test.go +++ b/netutil/net_example_test.go @@ -89,3 +89,68 @@ func ExampleEncodeUrl() { // Output: // http://www.lancet.com?a=1&b=%5B2%5D } + +func ExampleHttpClient_DecodeResponse() { + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos/1", + Method: "GET", + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil || resp.StatusCode != 200 { + return + } + + type Todo struct { + UserId int `json:"userId"` + Id int `json:"id"` + Title string `json:"title"` + Completed bool `json:"completed"` + } + + var todo Todo + err = httpClient.DecodeResponse(resp, &todo) + if err != nil { + return + } + + fmt.Println(todo.Id) + + // Output: + // 1 +} + +func ExampleStructToUrlValues() { + type TodoQuery struct { + Id int `json:"id"` + Name string `json:"name"` + } + todoQuery := TodoQuery{ + Id: 1, + Name: "Test", + } + todoValues := StructToUrlValues(todoQuery) + + fmt.Println(todoValues.Get("id")) + fmt.Println(todoValues.Get("name")) + + // Output: + // 1 + // Test +} + +func ExampleConvertMapToQueryString() { + var m = map[string]any{ + "c": 3, + "a": 1, + "b": 2, + } + + qs := ConvertMapToQueryString(m) + + fmt.Println(qs) + + // Output: + // a=1&b=2&c=3 +}