From ab81d9c283c8a1089c08cd81de69766d8c6e0884 Mon Sep 17 00:00:00 2001 From: dudaodong Date: Mon, 6 Mar 2023 17:23:54 +0800 Subject: [PATCH] fix: fix StructToUrlValues failed when tag contain omitempty --- docs/netutil.md | 28 +++-- docs/netutil_zh-CN.md | 28 +++-- netutil/http.go | 229 +++++++++++++++++++++++++++++++++++- netutil/http_client.go | 228 ----------------------------------- netutil/http_client_test.go | 95 --------------- netutil/http_test.go | 86 ++++++++++++++ 6 files changed, 351 insertions(+), 343 deletions(-) delete mode 100644 netutil/http_client.go delete mode 100644 netutil/http_client_test.go diff --git a/docs/netutil.md b/docs/netutil.md index 7057a65..4c30aea 100644 --- a/docs/netutil.md +++ b/docs/netutil.md @@ -551,22 +551,34 @@ package main import ( "fmt" - "github.com/duke-git/lancet/netutil" + "github.com/duke-git/lancet/v2/netutil" ) func main() { type TodoQuery struct { - Id int `json:"id"` - UserId int `json:"userId"` + Id int `json:"id"` + UserId int `json:"userId"` + Name string `json:"name,omitempty"` + Status string } - todoQuery := TodoQuery{ + item := TodoQuery{ Id: 1, - UserId: 2, + UserId: 123, + Name: "test", + Status: "completed", } - todoValues := netutil.StructToUrlValues(todoQuery) + queryValues := netutil.StructToUrlValues(item) - fmt.Println(todoValues.Get("id")) //1 - fmt.Println(todoValues.Get("userId")) //2 + fmt.Println(todoValues.Get("id")) + fmt.Println(todoValues.Get("userId")) + fmt.Println(todoValues.Get("name")) + fmt.Println(todoValues.Get("status")) + + // Output: + // 1 + // 123 + // test + // } ``` diff --git a/docs/netutil_zh-CN.md b/docs/netutil_zh-CN.md index ad6c4e1..1b5163b 100644 --- a/docs/netutil_zh-CN.md +++ b/docs/netutil_zh-CN.md @@ -550,22 +550,34 @@ package main import ( "fmt" - "github.com/duke-git/lancet/netutil" + "github.com/duke-git/lancet/v2/netutil" ) func main() { type TodoQuery struct { - Id int `json:"id"` - UserId int `json:"userId"` + Id int `json:"id"` + UserId int `json:"userId"` + Name string `json:"name,omitempty"` + Status string } - todoQuery := TodoQuery{ + item := TodoQuery{ Id: 1, - UserId: 2, + UserId: 123, + Name: "test", + Status: "completed", } - todoValues := netutil.StructToUrlValues(todoQuery) + queryValues := netutil.StructToUrlValues(item) - fmt.Println(todoValues.Get("id")) //1 - fmt.Println(todoValues.Get("userId")) //2 + fmt.Println(todoValues.Get("id")) + fmt.Println(todoValues.Get("userId")) + fmt.Println(todoValues.Get("name")) + fmt.Println(todoValues.Get("status")) + + // Output: + // 1 + // 123 + // test + // } ``` diff --git a/netutil/http.go b/netutil/http.go index c693592..9e93edf 100644 --- a/netutil/http.go +++ b/netutil/http.go @@ -13,30 +13,39 @@ package netutil import ( + "bytes" + "crypto/tls" "encoding/json" "errors" "fmt" + "io/ioutil" "net/http" + "net/url" + "reflect" + "regexp" "sort" "strings" + "time" + + "github.com/duke-git/lancet/slice" ) -//HttpGet send get http request +// HttpGet send get http request func HttpGet(url string, params ...interface{}) (*http.Response, error) { return doHttpRequest(http.MethodGet, url, params...) } -//HttpPost send post http request +// HttpPost send post http request func HttpPost(url string, params ...interface{}) (*http.Response, error) { return doHttpRequest(http.MethodPost, url, params...) } -//HttpPut send put http request +// HttpPut send put http request func HttpPut(url string, params ...interface{}) (*http.Response, error) { return doHttpRequest(http.MethodPut, url, params...) } -//HttpDelete send delete http request +// HttpDelete send delete http request func HttpDelete(url string, params ...interface{}) (*http.Response, error) { return doHttpRequest(http.MethodDelete, url, params...) } @@ -77,3 +86,215 @@ func ConvertMapToQueryString(param map[string]interface{}) string { } return build.String() } + +type HttpRequest struct { + RawURL string + Method string + Headers http.Header + QueryParams url.Values + FormData url.Values + Body []byte +} + +// HttpClientConfig contains some configurations for http client +type HttpClientConfig struct { + SSLEnabled bool + TLSConfig *tls.Config + Compressed bool + HandshakeTimeout time.Duration + ResponseTimeout time.Duration + Verbose bool +} + +// defaultHttpClientConfig defalut client config +var defaultHttpClientConfig = &HttpClientConfig{ + Compressed: false, + HandshakeTimeout: 20 * time.Second, + ResponseTimeout: 40 * time.Second, +} + +// HttpClient is used for sending http request +type HttpClient struct { + *http.Client + TLS *tls.Config + Request *http.Request + Config HttpClientConfig +} + +// NewHttpClient make a HttpClient instance +func NewHttpClient() *HttpClient { + client := &HttpClient{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSHandshakeTimeout: defaultHttpClientConfig.HandshakeTimeout, + ResponseHeaderTimeout: defaultHttpClientConfig.ResponseTimeout, + DisableCompression: !defaultHttpClientConfig.Compressed, + }, + }, + Config: *defaultHttpClientConfig, + } + + return client +} + +// NewHttpClientWithConfig make a HttpClient instance with pass config +func NewHttpClientWithConfig(config *HttpClientConfig) *HttpClient { + if config == nil { + config = defaultHttpClientConfig + } + + client := &HttpClient{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSHandshakeTimeout: config.HandshakeTimeout, + ResponseHeaderTimeout: config.ResponseTimeout, + DisableCompression: !config.Compressed, + }, + }, + Config: *config, + } + + if config.SSLEnabled { + client.TLS = config.TLSConfig + } + + return client +} + +// SendRequest send http request +func (client *HttpClient) SendRequest(request *HttpRequest) (*http.Response, error) { + err := validateRequest(request) + if err != nil { + return nil, err + } + + rawUrl := request.RawURL + + req, err := http.NewRequest(request.Method, rawUrl, bytes.NewBuffer(request.Body)) + if err != nil { + return nil, err + } + + client.setTLS(rawUrl) + client.setHeader(req, request.Headers) + client.setQueryParam(req, rawUrl, request.QueryParams) + + if request.FormData != nil { + client.setFormData(req, request.FormData) + } + + client.Request = req + + resp, err := client.Client.Do(req) + if err != nil { + return nil, err + } + + return resp, nil +} + +// DecodeResponse decode response into target object +func (client *HttpClient) DecodeResponse(resp *http.Response, target interface{}) error { + if resp == nil { + return errors.New("invalid target param") + } + defer resp.Body.Close() + return json.NewDecoder(resp.Body).Decode(target) +} + +// setTLS set http client transport TLSClientConfig +func (client *HttpClient) setTLS(rawUrl string) { + if strings.HasPrefix(rawUrl, "https") { + if transport, ok := client.Client.Transport.(*http.Transport); ok { + transport.TLSClientConfig = client.TLS + } + } +} + +// setHeader set http rquest header +func (client *HttpClient) setHeader(req *http.Request, headers http.Header) { + if headers == nil { + headers = make(http.Header) + } + + if _, ok := headers["Accept"]; !ok { + headers["Accept"] = []string{"*/*"} + } + if _, ok := headers["Accept-Encoding"]; !ok && client.Config.Compressed { + headers["Accept-Encoding"] = []string{"deflate, gzip"} + } + + req.Header = headers +} + +// setQueryParam set http request query string param +func (client *HttpClient) setQueryParam(req *http.Request, reqUrl string, queryParam url.Values) error { + if queryParam != nil { + if !strings.Contains(reqUrl, "?") { + reqUrl = reqUrl + "?" + queryParam.Encode() + } else { + reqUrl = reqUrl + "&" + queryParam.Encode() + } + u, err := url.Parse(reqUrl) + if err != nil { + return err + } + req.URL = u + } + return nil +} + +func (client *HttpClient) setFormData(req *http.Request, values url.Values) { + formData := []byte(values.Encode()) + req.Body = ioutil.NopCloser(bytes.NewReader(formData)) + req.ContentLength = int64(len(formData)) +} + +// validateRequest check if a request has url, and valid method. +func validateRequest(req *HttpRequest) error { + if req.RawURL == "" { + return errors.New("invalid request url") + } + + // common HTTP methods + methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", + "HEAD", "CONNECT", "OPTIONS", "TRACE"} + + if !slice.Contain(methods, strings.ToUpper(req.Method)) { + return errors.New("invalid request method") + } + + return nil +} + +// StructToUrlValues convert struct to url valuse, +// only convert the field which is exported and has `json` tag +func StructToUrlValues(targetStruct interface{}) url.Values { + rv := reflect.ValueOf(targetStruct) + rt := reflect.TypeOf(targetStruct) + + if rt.Kind() == reflect.Ptr { + rt = rt.Elem() + } + if rt.Kind() != reflect.Struct { + panic(fmt.Errorf("data type %T not support, shuld be struct or pointer to struct", targetStruct)) + } + + result := url.Values{} + + fieldNum := rt.NumField() + pattern := `^[A-Z]` + regex := regexp.MustCompile(pattern) + for i := 0; i < fieldNum; i++ { + name := rt.Field(i).Name + tag := rt.Field(i).Tag.Get("json") + if regex.MatchString(name) && tag != "" { + if strings.Contains(tag, "omitempty") { + tag = strings.Split(tag, ",")[0] + } + result.Add(tag, fmt.Sprintf("%v", rv.Field(i).Interface())) + } + } + + return result +} diff --git a/netutil/http_client.go b/netutil/http_client.go deleted file mode 100644 index 3e97d2c..0000000 --- a/netutil/http_client.go +++ /dev/null @@ -1,228 +0,0 @@ -package netutil - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "reflect" - "regexp" - "strings" - "time" - - "github.com/duke-git/lancet/slice" -) - -// HttpRequest struct is a composed http request -type HttpRequest struct { - RawURL string - Method string - Headers http.Header - QueryParams url.Values - FormData url.Values - Body []byte -} - -// HttpClientConfig contains some configurations for http client -type HttpClientConfig struct { - SSLEnabled bool - TLSConfig *tls.Config - Compressed bool - HandshakeTimeout time.Duration - ResponseTimeout time.Duration - Verbose bool -} - -// defaultHttpClientConfig defalut client config -var defaultHttpClientConfig = &HttpClientConfig{ - Compressed: false, - HandshakeTimeout: 20 * time.Second, - ResponseTimeout: 40 * time.Second, -} - -// HttpClient is used for sending http request -type HttpClient struct { - *http.Client - TLS *tls.Config - Request *http.Request - Config HttpClientConfig -} - -// NewHttpClient make a HttpClient instance -func NewHttpClient() *HttpClient { - client := &HttpClient{ - Client: &http.Client{ - Transport: &http.Transport{ - TLSHandshakeTimeout: defaultHttpClientConfig.HandshakeTimeout, - ResponseHeaderTimeout: defaultHttpClientConfig.ResponseTimeout, - DisableCompression: !defaultHttpClientConfig.Compressed, - }, - }, - Config: *defaultHttpClientConfig, - } - - return client -} - -// NewHttpClientWithConfig make a HttpClient instance with pass config -func NewHttpClientWithConfig(config *HttpClientConfig) *HttpClient { - if config == nil { - config = defaultHttpClientConfig - } - - client := &HttpClient{ - Client: &http.Client{ - Transport: &http.Transport{ - TLSHandshakeTimeout: config.HandshakeTimeout, - ResponseHeaderTimeout: config.ResponseTimeout, - DisableCompression: !config.Compressed, - }, - }, - Config: *config, - } - - if config.SSLEnabled { - client.TLS = config.TLSConfig - } - - return client -} - -// SendRequest send http request -func (client *HttpClient) SendRequest(request *HttpRequest) (*http.Response, error) { - err := validateRequest(request) - if err != nil { - return nil, err - } - - rawUrl := request.RawURL - - req, err := http.NewRequest(request.Method, rawUrl, bytes.NewBuffer(request.Body)) - if err != nil { - return nil, err - } - - client.setTLS(rawUrl) - client.setHeader(req, request.Headers) - client.setQueryParam(req, rawUrl, request.QueryParams) - - if request.FormData != nil { - client.setFormData(req, request.FormData) - } - - client.Request = req - - resp, err := client.Client.Do(req) - if err != nil { - return nil, err - } - - return resp, nil -} - -// DecodeResponse decode response into target object -func (client *HttpClient) DecodeResponse(resp *http.Response, target interface{}) error { - if resp == nil { - return errors.New("invalid target param") - } - defer resp.Body.Close() - return json.NewDecoder(resp.Body).Decode(target) -} - -// setTLS set http client transport TLSClientConfig -func (client *HttpClient) setTLS(rawUrl string) { - if strings.HasPrefix(rawUrl, "https") { - if transport, ok := client.Client.Transport.(*http.Transport); ok { - transport.TLSClientConfig = client.TLS - } - } -} - -// setHeader set http rquest header -func (client *HttpClient) setHeader(req *http.Request, headers http.Header) { - if headers == nil { - headers = make(http.Header) - } - - if _, ok := headers["Accept"]; !ok { - headers["Accept"] = []string{"*/*"} - } - if _, ok := headers["Accept-Encoding"]; !ok && client.Config.Compressed { - headers["Accept-Encoding"] = []string{"deflate, gzip"} - } - - req.Header = headers -} - -// setQueryParam set http request query string param -func (client *HttpClient) setQueryParam(req *http.Request, reqUrl string, queryParam url.Values) error { - if queryParam != nil { - if !strings.Contains(reqUrl, "?") { - reqUrl = reqUrl + "?" + queryParam.Encode() - } else { - reqUrl = reqUrl + "&" + queryParam.Encode() - } - u, err := url.Parse(reqUrl) - if err != nil { - return err - } - req.URL = u - } - return nil -} - -func (client *HttpClient) setFormData(req *http.Request, values url.Values) { - formData := []byte(values.Encode()) - req.Body = ioutil.NopCloser(bytes.NewReader(formData)) - req.ContentLength = int64(len(formData)) -} - -// validateRequest check if a request has url, and valid method. -func validateRequest(req *HttpRequest) error { - if req.RawURL == "" { - return errors.New("invalid request url") - } - - // common HTTP methods - methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", - "HEAD", "CONNECT", "OPTIONS", "TRACE"} - - if !slice.Contain(methods, strings.ToUpper(req.Method)) { - return errors.New("invalid request method") - } - - return nil -} - -// StructToUrlValues convert struct to url valuse, -// only convert the field which is exported and has `json` tag -func StructToUrlValues(targetStruct interface{}) url.Values { - rv := reflect.ValueOf(targetStruct) - rt := reflect.TypeOf(targetStruct) - - if rt.Kind() == reflect.Ptr { - rt = rt.Elem() - } - if rt.Kind() != reflect.Struct { - panic(fmt.Errorf("data type %T not support, shuld be struct or pointer to struct", targetStruct)) - } - - result := url.Values{} - - fieldNum := rt.NumField() - pattern := `^[A-Z]` - regex := regexp.MustCompile(pattern) - for i := 0; i < fieldNum; i++ { - name := rt.Field(i).Name - tag := rt.Field(i).Tag.Get("json") - if regex.MatchString(name) && tag != "" { - result.Add(tag, fmt.Sprintf("%v", rv.Field(i).Interface())) - } - } - - return result -} diff --git a/netutil/http_client_test.go b/netutil/http_client_test.go deleted file mode 100644 index fba2e5a..0000000 --- a/netutil/http_client_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package netutil - -import ( - "io/ioutil" - "log" - "net/http" - "net/url" - "testing" - - "github.com/duke-git/lancet/internal" -) - -func TestHttpClient_Get(t *testing.T) { - assert := internal.NewAssert(t, "TestHttpClient_Get") - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos/1", - Method: "GET", - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - log.Fatal(err) - } - - type Todo struct { - UserId int `json:"userId"` - Id int `json:"id"` - Title string `json:"title"` - Completed bool `json:"completed"` - } - - var todo Todo - httpClient.DecodeResponse(resp, &todo) - - assert.Equal(1, todo.Id) -} - -func TestHttpClent_Post(t *testing.T) { - header := http.Header{} - header.Add("Content-Type", "multipart/form-data") - - postData := url.Values{} - postData.Add("userId", "1") - postData.Add("title", "testItem") - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos", - Method: "POST", - Headers: header, - FormData: postData, - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil { - log.Fatal(err) - } - - body, _ := ioutil.ReadAll(resp.Body) - t.Log("response: ", resp.StatusCode, string(body)) -} - -func TestStructToUrlValues(t *testing.T) { - assert := internal.NewAssert(t, "TestStructToUrlValues") - - type TodoQuery struct { - Id int `json:"id"` - UserId int `json:"userId"` - } - todoQuery := TodoQuery{ - Id: 1, - UserId: 1, - } - todoValues := StructToUrlValues(todoQuery) - - assert.Equal("1", todoValues.Get("id")) - assert.Equal("1", todoValues.Get("userId")) - - request := &HttpRequest{ - RawURL: "https://jsonplaceholder.typicode.com/todos", - Method: "GET", - QueryParams: todoValues, - } - - httpClient := NewHttpClient() - resp, err := httpClient.SendRequest(request) - if err != nil || resp.StatusCode != 200 { - log.Fatal(err) - } - - body, _ := ioutil.ReadAll(resp.Body) - t.Log("response: ", string(body)) -} diff --git a/netutil/http_test.go b/netutil/http_test.go index 8a683cc..ebcfa24 100644 --- a/netutil/http_test.go +++ b/netutil/http_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io/ioutil" "log" + "net/http" "net/url" "testing" @@ -163,3 +164,88 @@ func TestParseResponse(t *testing.T) { } t.Log("response: ", toDoResp) } + +func TestHttpClient_Get(t *testing.T) { + assert := internal.NewAssert(t, "TestHttpClient_Get") + + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos/1", + Method: "GET", + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil || resp.StatusCode != 200 { + log.Fatal(err) + } + + type Todo struct { + UserId int `json:"userId"` + Id int `json:"id"` + Title string `json:"title"` + Completed bool `json:"completed"` + } + + var todo Todo + httpClient.DecodeResponse(resp, &todo) + + assert.Equal(1, todo.Id) +} + +func TestHttpClent_Post(t *testing.T) { + header := http.Header{} + header.Add("Content-Type", "multipart/form-data") + + postData := url.Values{} + postData.Add("userId", "1") + postData.Add("title", "testItem") + + request := &HttpRequest{ + RawURL: "https://jsonplaceholder.typicode.com/todos", + Method: "POST", + Headers: header, + FormData: postData, + } + + httpClient := NewHttpClient() + resp, err := httpClient.SendRequest(request) + if err != nil { + log.Fatal(err) + } + + body, _ := ioutil.ReadAll(resp.Body) + t.Log("response: ", resp.StatusCode, string(body)) +} + +func TestStructToUrlValues(t *testing.T) { + assert := internal.NewAssert(t, "TestStructToUrlValues") + + type TodoQuery struct { + Id int `json:"id"` + UserId int `json:"userId"` + Name string `json:"name,omitempty"` + Status string + } + item1 := TodoQuery{ + Id: 1, + UserId: 123, + Name: "test", + Status: "completed", + } + queryValues1 := StructToUrlValues(item1) + + assert.Equal("1", queryValues1.Get("id")) + assert.Equal("123", queryValues1.Get("userId")) + assert.Equal("test", queryValues1.Get("name")) + assert.Equal("", queryValues1.Get("status")) + + item2 := TodoQuery{ + Id: 2, + UserId: 456, + } + queryValues2 := StructToUrlValues(item2) + + assert.Equal("2", queryValues2.Get("id")) + assert.Equal("456", queryValues2.Get("userId")) + assert.Equal("", queryValues2.Get("name")) +}