cloudpods/pkg/util/httputils/httputils.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package httputils

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/moul/http2curl"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/gotypes"
	"yunion.io/x/pkg/trace"

	"yunion.io/x/onecloud/pkg/appctx"
)

type THttpMethod string

const (
	USER_AGENT = "yunioncloud-go/201708"

	GET    = THttpMethod("GET")
	HEAD   = THttpMethod("HEAD")
	POST   = THttpMethod("POST")
	PUT    = THttpMethod("PUT")
	PATCH  = THttpMethod("PATCH")
	DELETE = THttpMethod("DELETE")
	OPTION = THttpMethod("OPTION")

	IdleConnTimeout       = 60
	TLSHandshakeTimeout   = 10
	ResponseHeaderTimeout = 30
)

var (
	red    = color.New(color.FgRed, color.Bold).PrintlnFunc()
	green  = color.New(color.FgGreen, color.Bold).PrintlnFunc()
	yellow = color.New(color.FgYellow, color.Bold).PrintlnFunc()
	cyan   = color.New(color.FgHiCyan, color.Bold).PrintlnFunc()
)

type Error struct {
	Id     string
	Fields []string
}

type JSONClientError struct {
	Code    int
	Class   string
	Details string
	Data    Error
}

type JSONClientErrorMsg struct {
	Error *JSONClientError
}

func (e *JSONClientError) Error() string {
	errMsg := JSONClientErrorMsg{Error: e}
	return jsonutils.Marshal(errMsg).String()
}

func (err *JSONClientError) Cause() error {
	if len(err.Class) > 0 {
		return errors.Error(err.Class)
	} else if err.Code >= 500 {
		return errors.ErrServer
	} else if err.Code >= 400 {
		return errors.ErrClient
	} else {
		return errors.ErrUnclassified
	}
}

func ErrorCode(err error) int {
	if err == nil {
		return 0
	}
	switch je := err.(type) {
	case *JSONClientError:
		return je.Code
	}
	return -1
}

func ErrorMsg(err error) string {
	if err == nil {
		return ""
	}
	switch je := err.(type) {
	case *JSONClientError:
		return je.Details
	}
	return err.Error()
}

func GetAddrPort(urlStr string) (string, int, error) {
	parts, err := url.Parse(urlStr)
	if err != nil {
		return "", 0, err
	}
	host := parts.Host
	commaPos := strings.IndexByte(host, ':')
	if commaPos > 0 {
		port, err := strconv.ParseInt(host[commaPos+1:], 10, 32)
		if err != nil {
			return "", 0, err
		} else {
			return host[:commaPos], int(port), nil
		}
	} else {
		switch parts.Scheme {
		case "http":
			return parts.Host, 80, nil
		case "https":
			return parts.Host, 443, nil
		default:
			return "", 0, fmt.Errorf("Unknown schema %s", parts.Scheme)
		}
	}
}

func GetTransport(insecure bool) *http.Transport {
	return getTransport(insecure, false)
}

func adptiveDial(ctx context.Context, network, addr string) (net.Conn, error) {
	conn, err := net.DialTimeout(network, addr, 10*time.Second)
	if err != nil {
		return nil, err
	}
	return getConnDelegate(conn, 10*time.Second, 20*time.Second), nil
}

func getTransport(insecure bool, adaptive bool) *http.Transport {
	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		// 一个空闲连接保持连接的时间
		// IdleConnTimeout is the maximum amount of time an idle
		// (keep-alive) connection will remain idle before closing
		// itself.
		// Zero means no limit.
		IdleConnTimeout: IdleConnTimeout * time.Second,
		// 建立TCP连接后，等待TLS握手的超时时间
		// TLSHandshakeTimeout specifies the maximum amount of time waiting to
		// wait for a TLS handshake. Zero means no timeout.
		TLSHandshakeTimeout: TLSHandshakeTimeout * time.Second,
		// 发送请求后，等待服务端http响应的超时时间
		// ResponseHeaderTimeout, if non-zero, specifies the amount of
		// time to wait for a server's response headers after fully
		// writing the request (including its body, if any). This
		// time does not include the time to read the response body.
		ResponseHeaderTimeout: ResponseHeaderTimeout * time.Second,
		// 当请求携带Expect: 100-continue时，等待服务端100响应的超时时间
		// ExpectContinueTimeout, if non-zero, specifies the amount of
		// time to wait for a server's first response headers after fully
		// writing the request headers if the request has an
		// "Expect: 100-continue" header. Zero means no timeout and
		// causes the body to be sent immediately, without
		// waiting for the server to approve.
		// This time does not include the time to send the request header.
		ExpectContinueTimeout: 5 * time.Second,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: insecure},
	}
	if adaptive {
		tr.DialContext = adptiveDial
	} else {
		tr.DialContext = (&net.Dialer{
			// 建立TCP连接超时时间
			// Timeout is the maximum amount of time a dial will wait for
			// a connect to complete. If Deadline is also set, it may fail
			// earlier.
			//
			// The default is no timeout.
			//
			// When using TCP and dialing a host name with multiple IP
			// addresses, the timeout may be divided between them.
			//
			// With or without a timeout, the operating system may impose
			// its own earlier timeout. For instance, TCP timeouts are
			// often around 3 minutes.
			Timeout: 10 * time.Second,
			//
			// KeepAlive specifies the interval between keep-alive
			// probes for an active network connection.
			// If zero, keep-alive probes are sent with a default value
			// (currently 15 seconds), if supported by the protocol and operating
			// system. Network protocols or operating systems that do
			// not support keep-alives ignore this field.
			// If negative, keep-alive probes are disabled.
			KeepAlive: 5 * time.Second, // send keep-alive probe every 5 seconds
		}).DialContext
	}
	return tr
}

func GetClient(insecure bool, timeout time.Duration) *http.Client {
	adaptive := false
	if timeout == 0 {
		adaptive = true
	}
	tr := getTransport(insecure, adaptive)
	return &http.Client{
		Transport: tr,
		// 一个完整http request的超时时间
		// Timeout specifies a time limit for requests made by this
		// Client. The timeout includes connection time, any
		// redirects, and reading the response body. The timer remains
		// running after Get, Head, Post, or Do return and will
		// interrupt reading of the Response.Body.
		//
		// A Timeout of zero means no timeout.
		//
		// The Client cancels requests to the underlying Transport
		// as if the Request's Context ended.
		//
		// For compatibility, the Client will also use the deprecated
		// CancelRequest method on Transport if found. New
		// RoundTripper implementations should use the Request's Context
		// for cancellation instead of implementing CancelRequest.
		Timeout: timeout,
	}
}

type TransportProxyFunc func(*http.Request) (*url.URL, error)

func SetClientProxyFunc(
	client *http.Client,
	proxyFunc TransportProxyFunc,
) bool {
	set := false
	if transport, ok := client.Transport.(*http.Transport); ok {
		transport.Proxy = proxyFunc
		set = true
	}
	return set
}

func GetTimeoutClient(timeout time.Duration) *http.Client {
	return GetClient(true, timeout)
}

func GetAdaptiveTimeoutClient() *http.Client {
	return GetClient(true, 0)
}

var defaultHttpClient *http.Client

func init() {
	defaultHttpClient = GetClient(true, time.Second*15)
}

func GetDefaultClient() *http.Client {
	return defaultHttpClient
}

func Request(client *http.Client, ctx context.Context, method THttpMethod, urlStr string, header http.Header, body io.Reader, debug bool) (*http.Response, error) {
	if client == nil {
		client = defaultHttpClient
	}
	if header == nil {
		header = http.Header{}
	}
	ctxData := appctx.FetchAppContextData(ctx)
	var clientTrace *trace.STrace
	if !ctxData.Trace.IsZero() {
		addr, port, err := GetAddrPort(urlStr)
		if err != nil {
			return nil, err
		}
		clientTrace = trace.StartClientTrace(&ctxData.Trace, addr, port, ctxData.ServiceName)
		clientTrace.AddClientRequestHeader(header)
	}
	if len(ctxData.RequestId) > 0 {
		header.Set("X-Request-Id", ctxData.RequestId)
	}
	req, err := http.NewRequest(string(method), urlStr, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", USER_AGENT)
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Accept-Encoding", "*")
	if body == nil {
		if method != GET && method != HEAD {
			req.ContentLength = 0
			req.Header.Set("Content-Length", "0")
		}
	} else {
		clen := header.Get("Content-Length")
		if len(clen) > 0 {
			req.ContentLength, _ = strconv.ParseInt(clen, 10, 64)
		}
	}
	if header != nil {
		for k, vs := range header {
			for i, v := range vs {
				if i == 0 {
					req.Header.Set(k, v)
				} else {
					req.Header.Add(k, v)
				}
			}
		}
	}
	if debug {
		dump, _ := httputil.DumpRequestOut(req, false)
		yellow(string(dump))
		// 忽略掉上传文件的请求,避免大量日志输出
		if header.Get("Content-Type") != "application/octet-stream" {
			curlCmd, _ := http2curl.GetCurlCommand(req)
			cyan("CURL:", curlCmd, "\n")
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		red(err.Error())
	}
	if err == nil && clientTrace != nil {
		clientTrace.EndClientTraceHeader(resp.Header)
	}
	return resp, err
}

func JSONRequest(client *http.Client, ctx context.Context, method THttpMethod, urlStr string, header http.Header, body jsonutils.JSONObject, debug bool) (http.Header, jsonutils.JSONObject, error) {
	var bodystr string
	if !gotypes.IsNil(body) {
		bodystr = body.String()
	}
	jbody := strings.NewReader(bodystr)
	if header == nil {
		header = http.Header{}
	}
	header.Set("Content-Length", strconv.FormatInt(int64(len(bodystr)), 10))
	header.Set("Content-Type", "application/json")
	resp, err := Request(client, ctx, method, urlStr, header, jbody, debug)
	return ParseJSONResponse(resp, err, debug)
}

// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//
// Subsequently this allows golang http RoundTripper
// to re-use the same connection for future requests.
func CloseResponse(resp *http.Response) {
	// Callers should close resp.Body when done reading from it.
	// If resp.Body is not closed, the Client's underlying RoundTripper
	// (typically Transport) may not be able to re-use a persistent TCP
	// connection to the server for a subsequent "keep-alive" request.
	if resp != nil && resp.Body != nil {
		// Drain any remaining Body and then close the connection.
		// Without this closing connection would disallow re-using
		// the same connection for future uses.
		//  - http://stackoverflow.com/a/17961593/4465767
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

func ParseResponse(resp *http.Response, err error, debug bool) (http.Header, []byte, error) {
	if err != nil {
		ce := JSONClientError{}
		ce.Code = 499
		ce.Details = err.Error()
		return nil, nil, &ce
	}
	defer CloseResponse(resp)
	if debug {
		dump, _ := httputil.DumpResponse(resp, false)
		if resp.StatusCode < 300 {
			green(string(dump))
		} else if resp.StatusCode < 400 {
			yellow(string(dump))
		} else {
			red(string(dump))
		}
	}
	rbody, err := ioutil.ReadAll(resp.Body)
	if debug {
		fmt.Fprintf(os.Stderr, "Response body: %s\n", string(rbody))
	}
	if err != nil {
		return nil, nil, fmt.Errorf("Fail to read body: %s", err)
	}
	if resp.StatusCode < 300 {
		return resp.Header, rbody, nil
	} else if resp.StatusCode >= 300 && resp.StatusCode < 400 {
		ce := JSONClientError{}
		ce.Code = resp.StatusCode
		ce.Details = resp.Header.Get("Location")
		ce.Class = "redirect"
		return nil, nil, &ce
	} else {
		ce := JSONClientError{}
		ce.Code = resp.StatusCode
		ce.Details = resp.Status
		if len(rbody) > 0 {
			ce.Details = string(rbody)
		}
		return nil, nil, &ce
	}
}

func ParseJSONResponse(resp *http.Response, err error, debug bool) (http.Header, jsonutils.JSONObject, error) {
	if err != nil {
		ce := JSONClientError{}
		ce.Code = 499
		ce.Details = err.Error()
		return nil, nil, &ce
	}
	defer CloseResponse(resp)
	if debug {
		dump, _ := httputil.DumpResponse(resp, false)
		if resp.StatusCode < 300 {
			green(string(dump))
		} else if resp.StatusCode < 400 {
			yellow(string(dump))
		} else {
			red(string(dump))
		}
	}
	rbody, err := ioutil.ReadAll(resp.Body)
	if debug {
		fmt.Fprintf(os.Stderr, "Response body: %s\n", string(rbody))
	}
	if err != nil {
		return nil, nil, fmt.Errorf("Fail to read body: %s", err)
	}

	var jrbody jsonutils.JSONObject = nil
	if len(rbody) > 0 && string(rbody[0]) == "{" {
		var err error
		jrbody, err = jsonutils.Parse(rbody)
		if err != nil && debug {
			fmt.Fprintf(os.Stderr, "parsing json failed: %s", err)
		}
	}

	if resp.StatusCode < 300 {
		return resp.Header, jrbody, nil
	} else if resp.StatusCode >= 300 && resp.StatusCode < 400 {
		ce := JSONClientError{}
		ce.Code = resp.StatusCode
		ce.Details = resp.Header.Get("Location")
		ce.Class = "redirect"
		return nil, nil, &ce
	} else {
		ce := JSONClientError{}

		if jrbody == nil {
			ce.Code = resp.StatusCode
			ce.Details = resp.Status
			if len(rbody) > 0 {
				ce.Details = string(rbody)
			}
			return nil, nil, &ce
		}

		err = jrbody.Unmarshal(&ce)
		if len(ce.Class) > 0 && ce.Code >= 400 && len(ce.Details) > 0 {
			return nil, nil, &ce
		}

		jrbody1, err := jrbody.GetMap()
		if err != nil {
			err = jrbody.Unmarshal(&ce)
			if err != nil {
				ce.Details = err.Error()
			}
			return nil, nil, &ce
		}
		var jrbody2 jsonutils.JSONObject
		if len(jrbody1) > 1 {
			jrbody2 = jsonutils.Marshal(jrbody1)
		} else {
			for _, v := range jrbody1 {
				jrbody2 = v
			}
		}
		if ecode, _ := jrbody2.GetString("code"); len(ecode) > 0 {
			code, err := strconv.Atoi(ecode)
			if err != nil {
				ce.Class = ecode
			} else {
				ce.Code = code
			}
		}
		if ce.Code == 0 {
			ce.Code = resp.StatusCode
		}
		if edetail := jsonutils.GetAnyString(jrbody2, []string{"message", "detail", "details", "error_msg"}); len(edetail) > 0 {
			ce.Details = edetail
		}
		if eclass := jsonutils.GetAnyString(jrbody2, []string{"title", "type", "error_code"}); len(eclass) > 0 {
			ce.Class = eclass
		}
		return nil, nil, &ce
	}
}

func JoinPath(ep string, path string) string {
	return strings.TrimRight(ep, "/") + "/" + strings.TrimLeft(path, "/")
}