mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-17 07:08:01 +08:00
DecodeResponseBody used zstd.NewReader with its default options, which hands blocks to up to four asynchronous decoders: allocations and handoff latency on every streamed response for no gain on bodies read as they arrive. Decode with WithDecoderConcurrency(1) instead. A response without Content-Encoding was sniffed for gzip/zstd magic with Peek(4) before DecodeResponseBody returned, blocking the caller until the upstream sent its first bytes. Detect on the first Read instead, waiting for more than one byte only when the first byte can start a gzip or zstd frame.
224 lines
6.7 KiB
Go
224 lines
6.7 KiB
Go
package helps
|
|
|
|
import (
|
|
"bufio"
|
|
"compress/flate"
|
|
"compress/gzip"
|
|
"compress/zlib"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/andybalholm/brotli"
|
|
"github.com/klauspost/compress/zstd"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type compositeReadCloser struct {
|
|
io.Reader
|
|
closers []func() error
|
|
}
|
|
|
|
func (c *compositeReadCloser) Close() error {
|
|
var firstErr error
|
|
for i := range c.closers {
|
|
if c.closers[i] == nil {
|
|
continue
|
|
}
|
|
if errClose := c.closers[i](); errClose != nil && firstErr == nil {
|
|
firstErr = errClose
|
|
}
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
// peekableBody wraps a bufio.Reader around the original ReadCloser so that
|
|
// magic bytes can be inspected without consuming them from the stream.
|
|
type peekableBody struct {
|
|
*bufio.Reader
|
|
closer io.Closer
|
|
}
|
|
|
|
func (p *peekableBody) Close() error {
|
|
return p.closer.Close()
|
|
}
|
|
|
|
// DecodeResponseBody decodes declared encodings in reverse order, or sniffs gzip/zstd without a header.
|
|
// Closing the returned body also closes all decoders and the original body, including on setup errors.
|
|
func DecodeResponseBody(body io.ReadCloser, header http.Header) (io.ReadCloser, error) {
|
|
contentEncoding := strings.Join(header.Values("Content-Encoding"), ",")
|
|
if body == nil {
|
|
return nil, fmt.Errorf("response body is nil")
|
|
}
|
|
if contentEncoding == "" {
|
|
// No Content-Encoding header. Attempt best-effort magic-byte detection to
|
|
// handle misbehaving upstreams that compress without setting the header.
|
|
// Only gzip (1f 8b) and zstd (28 b5 2f fd) have reliable magic sequences;
|
|
// br and deflate have none and are left as-is.
|
|
// The bufio wrapper preserves unread bytes so callers always see the full
|
|
// stream regardless of whether decompression was applied.
|
|
// Detection waits for the first Read: a streaming upstream may send headers long
|
|
// before its first byte, and peeking here would block the caller until then.
|
|
return &sniffedResponseBody{body: &peekableBody{Reader: bufio.NewReader(body), closer: body}}, nil
|
|
}
|
|
encodings := strings.Split(contentEncoding, ",")
|
|
reader := io.Reader(body)
|
|
decoderClosers := make([]func() error, 0, len(encodings))
|
|
cleanup := func() {
|
|
for i := len(decoderClosers) - 1; i >= 0; i-- {
|
|
if errClose := decoderClosers[i](); errClose != nil {
|
|
log.Errorf("response decoder: close decoder error: %v", errClose)
|
|
}
|
|
}
|
|
if errClose := body.Close(); errClose != nil {
|
|
log.Errorf("response decoder: close body error: %v", errClose)
|
|
}
|
|
}
|
|
for index := len(encodings) - 1; index >= 0; index-- {
|
|
encoding := strings.TrimSpace(strings.ToLower(encodings[index]))
|
|
switch encoding {
|
|
case "", "identity":
|
|
continue
|
|
case "gzip":
|
|
gzipReader, errGzip := gzip.NewReader(reader)
|
|
if errGzip != nil {
|
|
cleanup()
|
|
return nil, fmt.Errorf("failed to create gzip reader: %w", errGzip)
|
|
}
|
|
reader = gzipReader
|
|
decoderClosers = append(decoderClosers, gzipReader.Close)
|
|
case "deflate":
|
|
deflateReader, errDeflate := newResponseDeflateReader(reader)
|
|
if errDeflate != nil {
|
|
cleanup()
|
|
return nil, errDeflate
|
|
}
|
|
reader = deflateReader
|
|
decoderClosers = append(decoderClosers, deflateReader.Close)
|
|
case "br":
|
|
reader = brotli.NewReader(reader)
|
|
case "zstd":
|
|
decoder, errZstd := newResponseZstdReader(reader)
|
|
if errZstd != nil {
|
|
cleanup()
|
|
return nil, fmt.Errorf("failed to create zstd reader: %w", errZstd)
|
|
}
|
|
reader = decoder
|
|
decoderClosers = append(decoderClosers, func() error {
|
|
decoder.Close()
|
|
return nil
|
|
})
|
|
default:
|
|
cleanup()
|
|
return nil, fmt.Errorf("unsupported content encoding %q", encoding)
|
|
}
|
|
}
|
|
if len(decoderClosers) == 0 && reader == body {
|
|
return body, nil
|
|
}
|
|
closers := make([]func() error, 0, len(decoderClosers)+1)
|
|
for index := len(decoderClosers) - 1; index >= 0; index-- {
|
|
closers = append(closers, decoderClosers[index])
|
|
}
|
|
closers = append(closers, body.Close)
|
|
return &compositeReadCloser{Reader: reader, closers: closers}, nil
|
|
}
|
|
|
|
func newResponseDeflateReader(reader io.Reader) (io.ReadCloser, error) {
|
|
buffered := bufio.NewReader(reader)
|
|
header, errPeek := buffered.Peek(2)
|
|
if errPeek == nil && isZlibHeader(header) {
|
|
zlibReader, errZlib := zlib.NewReader(buffered)
|
|
if errZlib != nil {
|
|
return nil, fmt.Errorf("failed to create zlib deflate reader: %w", errZlib)
|
|
}
|
|
return zlibReader, nil
|
|
}
|
|
return flate.NewReader(buffered), nil
|
|
}
|
|
|
|
func isZlibHeader(header []byte) bool {
|
|
if len(header) < 2 {
|
|
return false
|
|
}
|
|
cmf, flg := header[0], header[1]
|
|
return cmf&0x0f == 8 && cmf>>4 <= 7 && (uint16(cmf)<<8|uint16(flg))%31 == 0
|
|
}
|
|
|
|
// newResponseZstdReader decodes one response synchronously. The default decoder hands blocks to up
|
|
// to four asynchronous block decoders, which costs allocations and handoff latency and gains
|
|
// nothing on bodies that are read as they arrive.
|
|
func newResponseZstdReader(reader io.Reader) (*zstd.Decoder, error) {
|
|
return zstd.NewReader(reader, zstd.WithDecoderConcurrency(1))
|
|
}
|
|
|
|
// sniffedResponseBody decodes a body that arrived without Content-Encoding but starts with
|
|
// gzip or zstd magic bytes. Detection runs on the first Read and waits for more than one byte
|
|
// only when the first byte can start a gzip or zstd frame.
|
|
type sniffedResponseBody struct {
|
|
body *peekableBody
|
|
once sync.Once
|
|
mu sync.Mutex
|
|
reader io.Reader
|
|
decoder func() error
|
|
err error
|
|
}
|
|
|
|
func (s *sniffedResponseBody) detect() {
|
|
reader := io.Reader(s.body)
|
|
var closeDecoder func() error
|
|
defer func() {
|
|
s.mu.Lock()
|
|
s.reader, s.decoder = reader, closeDecoder
|
|
s.mu.Unlock()
|
|
}()
|
|
first, _ := s.body.Peek(1)
|
|
if len(first) == 0 || (first[0] != 0x1f && first[0] != 0x28) {
|
|
return
|
|
}
|
|
magic, errPeek := s.body.Peek(4)
|
|
if errPeek != nil && !(errPeek == io.EOF && len(magic) >= 2) {
|
|
return
|
|
}
|
|
switch {
|
|
case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b:
|
|
gzipReader, errGzip := gzip.NewReader(s.body)
|
|
if errGzip != nil {
|
|
s.err = fmt.Errorf("magic-byte gzip: failed to create reader: %w", errGzip)
|
|
return
|
|
}
|
|
reader, closeDecoder = gzipReader, gzipReader.Close
|
|
case len(magic) >= 4 && magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd:
|
|
decoder, errZstd := newResponseZstdReader(s.body)
|
|
if errZstd != nil {
|
|
s.err = fmt.Errorf("magic-byte zstd: failed to create reader: %w", errZstd)
|
|
return
|
|
}
|
|
reader, closeDecoder = decoder, func() error { decoder.Close(); return nil }
|
|
}
|
|
}
|
|
|
|
func (s *sniffedResponseBody) Read(p []byte) (int, error) {
|
|
s.once.Do(s.detect)
|
|
if s.err != nil {
|
|
return 0, s.err
|
|
}
|
|
return s.reader.Read(p)
|
|
}
|
|
|
|
func (s *sniffedResponseBody) Close() error {
|
|
s.mu.Lock()
|
|
closeDecoder := s.decoder
|
|
s.mu.Unlock()
|
|
var errDecoder error
|
|
if closeDecoder != nil {
|
|
errDecoder = closeDecoder()
|
|
}
|
|
if errClose := s.body.Close(); errClose != nil {
|
|
return errClose
|
|
}
|
|
return errDecoder
|
|
}
|