From 3c948a5808971ffe2b150414f36ec2fab0b671be Mon Sep 17 00:00:00 2001 From: JohnsonRan Date: Tue, 27 Oct 2026 04:18:42 +0900 Subject: [PATCH] perf(executor): decode upstream zstd synchronously and sniff unlabeled bodies lazily DecodeResponseBody used zstd.NewReader with its default options, which hands blocks to up to four asynchronous decoders: allocations and handoff latency on every streamed response for no gain on bodies read as they arrive. Decode with WithDecoderConcurrency(1) instead. A response without Content-Encoding was sniffed for gzip/zstd magic with Peek(4) before DecodeResponseBody returned, blocking the caller until the upstream sent its first bytes. Detect on the first Read instead, waiting for more than one byte only when the first byte can start a gzip or zstd frame. --- .../runtime/executor/helps/response_decode.go | 118 ++++++++++++------ .../helps/response_decode_bench_test.go | 117 +++++++++++++++++ .../executor/helps/response_decode_test.go | 54 ++++++++ 3 files changed, 251 insertions(+), 38 deletions(-) create mode 100644 internal/runtime/executor/helps/response_decode_bench_test.go diff --git a/internal/runtime/executor/helps/response_decode.go b/internal/runtime/executor/helps/response_decode.go index 09373dacfe..6f1419cc71 100644 --- a/internal/runtime/executor/helps/response_decode.go +++ b/internal/runtime/executor/helps/response_decode.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "strings" + "sync" "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" @@ -58,43 +59,9 @@ func DecodeResponseBody(body io.ReadCloser, header http.Header) (io.ReadCloser, // br and deflate have none and are left as-is. // The bufio wrapper preserves unread bytes so callers always see the full // stream regardless of whether decompression was applied. - pb := &peekableBody{Reader: bufio.NewReader(body), closer: body} - magic, peekErr := pb.Peek(4) - if peekErr == nil || (peekErr == io.EOF && len(magic) >= 2) { - switch { - case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: - gzipReader, gzErr := gzip.NewReader(pb) - if gzErr != nil { - if errClose := pb.Close(); errClose != nil { - log.Errorf("response decoder: close body error: %v", errClose) - } - return nil, fmt.Errorf("magic-byte gzip: failed to create reader: %w", gzErr) - } - return &compositeReadCloser{ - Reader: gzipReader, - closers: []func() error{ - gzipReader.Close, - pb.Close, - }, - }, nil - case len(magic) >= 4 && magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd: - decoder, zdErr := zstd.NewReader(pb) - if zdErr != nil { - if errClose := pb.Close(); errClose != nil { - log.Errorf("response decoder: close body error: %v", errClose) - } - return nil, fmt.Errorf("magic-byte zstd: failed to create reader: %w", zdErr) - } - return &compositeReadCloser{ - Reader: decoder, - closers: []func() error{ - func() error { decoder.Close(); return nil }, - pb.Close, - }, - }, nil - } - } - return pb, nil + // Detection waits for the first Read: a streaming upstream may send headers long + // before its first byte, and peeking here would block the caller until then. + return &sniffedResponseBody{body: &peekableBody{Reader: bufio.NewReader(body), closer: body}}, nil } encodings := strings.Split(contentEncoding, ",") reader := io.Reader(body) @@ -133,7 +100,7 @@ func DecodeResponseBody(body io.ReadCloser, header http.Header) (io.ReadCloser, case "br": reader = brotli.NewReader(reader) case "zstd": - decoder, errZstd := zstd.NewReader(reader) + decoder, errZstd := newResponseZstdReader(reader) if errZstd != nil { cleanup() return nil, fmt.Errorf("failed to create zstd reader: %w", errZstd) @@ -179,3 +146,78 @@ func isZlibHeader(header []byte) bool { cmf, flg := header[0], header[1] return cmf&0x0f == 8 && cmf>>4 <= 7 && (uint16(cmf)<<8|uint16(flg))%31 == 0 } + +// newResponseZstdReader decodes one response synchronously. The default decoder hands blocks to up +// to four asynchronous block decoders, which costs allocations and handoff latency and gains +// nothing on bodies that are read as they arrive. +func newResponseZstdReader(reader io.Reader) (*zstd.Decoder, error) { + return zstd.NewReader(reader, zstd.WithDecoderConcurrency(1)) +} + +// sniffedResponseBody decodes a body that arrived without Content-Encoding but starts with +// gzip or zstd magic bytes. Detection runs on the first Read and waits for more than one byte +// only when the first byte can start a gzip or zstd frame. +type sniffedResponseBody struct { + body *peekableBody + once sync.Once + mu sync.Mutex + reader io.Reader + decoder func() error + err error +} + +func (s *sniffedResponseBody) detect() { + reader := io.Reader(s.body) + var closeDecoder func() error + defer func() { + s.mu.Lock() + s.reader, s.decoder = reader, closeDecoder + s.mu.Unlock() + }() + first, _ := s.body.Peek(1) + if len(first) == 0 || (first[0] != 0x1f && first[0] != 0x28) { + return + } + magic, errPeek := s.body.Peek(4) + if errPeek != nil && !(errPeek == io.EOF && len(magic) >= 2) { + return + } + switch { + case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: + gzipReader, errGzip := gzip.NewReader(s.body) + if errGzip != nil { + s.err = fmt.Errorf("magic-byte gzip: failed to create reader: %w", errGzip) + return + } + reader, closeDecoder = gzipReader, gzipReader.Close + case len(magic) >= 4 && magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd: + decoder, errZstd := newResponseZstdReader(s.body) + if errZstd != nil { + s.err = fmt.Errorf("magic-byte zstd: failed to create reader: %w", errZstd) + return + } + reader, closeDecoder = decoder, func() error { decoder.Close(); return nil } + } +} + +func (s *sniffedResponseBody) Read(p []byte) (int, error) { + s.once.Do(s.detect) + if s.err != nil { + return 0, s.err + } + return s.reader.Read(p) +} + +func (s *sniffedResponseBody) Close() error { + s.mu.Lock() + closeDecoder := s.decoder + s.mu.Unlock() + var errDecoder error + if closeDecoder != nil { + errDecoder = closeDecoder() + } + if errClose := s.body.Close(); errClose != nil { + return errClose + } + return errDecoder +} diff --git a/internal/runtime/executor/helps/response_decode_bench_test.go b/internal/runtime/executor/helps/response_decode_bench_test.go new file mode 100644 index 0000000000..f7b5444e9a --- /dev/null +++ b/internal/runtime/executor/helps/response_decode_bench_test.go @@ -0,0 +1,117 @@ +package helps + +import ( + "bytes" + "fmt" + "io" + "math/rand" + "net/http" + "runtime" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" +) + +// benchZstdSSE returns a zstd-encoded body of about size bytes of Responses-style SSE deltas. +func benchZstdSSE(b *testing.B, size int) []byte { + b.Helper() + r := rand.New(rand.NewSource(1)) + words := strings.Fields("the function return error context request response model input output tool call result value index token cache stream event message content type text") + var plain strings.Builder + for i := 0; plain.Len() < size; i++ { + var delta strings.Builder + for j, n := 0, 4+r.Intn(12); j < n; j++ { + delta.WriteString(words[r.Intn(len(words))]) + delta.WriteByte(' ') + } + fmt.Fprintf(&plain, "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"sequence_number\":%d,\"delta\":%q}\n\n", i, delta.String()) + } + encoder, errNewWriter := zstd.NewWriter(nil) + if errNewWriter != nil { + b.Fatal(errNewWriter) + } + defer func() { _ = encoder.Close() }() + return encoder.EncodeAll([]byte(plain.String()), nil) +} + +var benchZstdHeader = http.Header{"Content-Encoding": {"zstd"}} + +// BenchmarkDecodeResponseBodyZstd decodes whole zstd upstream responses concurrently. +func BenchmarkDecodeResponseBodyZstd(b *testing.B) { + for _, size := range []int{64 << 10, 2 << 20} { + wire := benchZstdSSE(b, size) + b.Run(fmt.Sprintf("%dKiB", size>>10), func(b *testing.B) { + b.SetBytes(int64(size)) + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader) + if errDecode != nil { + b.Fatal(errDecode) + } + if _, errCopy := io.Copy(io.Discard, body); errCopy != nil { + b.Fatal(errCopy) + } + _ = body.Close() + } + }) + }) + } +} + +// BenchmarkDecodeResponseBodyOpenStreams holds 256 zstd responses open after their first read, +// like streams waiting on the model, and reports the heap held per open stream. +func BenchmarkDecodeResponseBodyOpenStreams(b *testing.B) { + const streams = 256 + wire := benchZstdSSE(b, 256<<10) + var heap float64 + for i := 0; i < b.N; i++ { + runtime.GC() + var before, during runtime.MemStats + runtime.ReadMemStats(&before) + bodies := make([]io.ReadCloser, streams) + buf := make([]byte, 512) + for s := range bodies { + body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader) + if errDecode != nil { + b.Fatal(errDecode) + } + if _, errRead := body.Read(buf); errRead != nil { + b.Fatal(errRead) + } + bodies[s] = body + } + runtime.GC() + runtime.ReadMemStats(&during) + heap += float64(int64(during.HeapInuse)-int64(before.HeapInuse)) / streams / 1024 + for _, body := range bodies { + _ = body.Close() + } + } + b.ReportMetric(heap/float64(b.N), "heap-KiB/stream") +} + +// BenchmarkDecodeResponseBodyUnlabeledFirstByte measures how long DecodeResponseBody blocks +// for a response without Content-Encoding whose first byte arrives 20 ms after the headers. +func BenchmarkDecodeResponseBodyUnlabeledFirstByte(b *testing.B) { + var blocked time.Duration + for i := 0; i < b.N; i++ { + reader, writer := io.Pipe() + go func() { + time.Sleep(20 * time.Millisecond) + _, _ = writer.Write([]byte("data: {}\n\n")) + _ = writer.Close() + }() + began := time.Now() + body, errDecode := DecodeResponseBody(reader, http.Header{}) + blocked += time.Since(began) + if errDecode != nil { + b.Fatal(errDecode) + } + _, _ = io.Copy(io.Discard, body) + _ = body.Close() + } + b.ReportMetric(float64(blocked.Milliseconds())/float64(b.N), "blocked-ms") +} diff --git a/internal/runtime/executor/helps/response_decode_test.go b/internal/runtime/executor/helps/response_decode_test.go index 999351bad1..bc08169a65 100644 --- a/internal/runtime/executor/helps/response_decode_test.go +++ b/internal/runtime/executor/helps/response_decode_test.go @@ -10,6 +10,7 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" @@ -131,3 +132,56 @@ func TestDecodeResponseBodyCompositeClose(t *testing.T) { t.Fatalf("close must attempt all closers and return first error: %v, order=%v", errClose, order) } } + +func TestDecodeResponseBodyUnlabeledDoesNotWaitForFirstByte(t *testing.T) { + reader, writer := io.Pipe() + returned := make(chan io.ReadCloser, 1) + go func() { + body, errDecode := DecodeResponseBody(reader, http.Header{}) + if errDecode != nil { + t.Error(errDecode) + } + returned <- body + }() + var body io.ReadCloser + select { + case body = <-returned: + case <-time.After(2 * time.Second): + t.Fatal("DecodeResponseBody blocked until the upstream sent its first byte") + } + go func() { + _, _ = writer.Write([]byte("data: {}\n\n")) + _ = writer.Close() + }() + got, errRead := io.ReadAll(body) + if errRead != nil || string(got) != "data: {}\n\n" { + t.Fatalf("read %q, %v", got, errRead) + } + if errClose := body.Close(); errClose != nil { + t.Fatal(errClose) + } +} + +func TestDecodeResponseBodySniffsSplitMagicBytes(t *testing.T) { + plain := []byte("data: response.completed\n\n") + for _, encoding := range []string{"gzip", "zstd"} { + wire := encodeResponseDecodeTestBody(t, encoding, plain) + reader, writer := io.Pipe() + go func() { + // The magic bytes arrive one write at a time. + for i := range wire { + _, _ = writer.Write(wire[i : i+1]) + } + _ = writer.Close() + }() + body, errDecode := DecodeResponseBody(reader, http.Header{}) + if errDecode != nil { + t.Fatal(errDecode) + } + got, errRead := io.ReadAll(body) + if errRead != nil || !bytes.Equal(got, plain) { + t.Fatalf("%s: read %q, %v", encoding, got, errRead) + } + _ = body.Close() + } +}