diff --git a/internal/runtime/executor/helps/response_decode.go b/internal/runtime/executor/helps/response_decode.go index 09373dacfe..6f1419cc71 100644 --- a/internal/runtime/executor/helps/response_decode.go +++ b/internal/runtime/executor/helps/response_decode.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "strings" + "sync" "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" @@ -58,43 +59,9 @@ func DecodeResponseBody(body io.ReadCloser, header http.Header) (io.ReadCloser, // br and deflate have none and are left as-is. // The bufio wrapper preserves unread bytes so callers always see the full // stream regardless of whether decompression was applied. - pb := &peekableBody{Reader: bufio.NewReader(body), closer: body} - magic, peekErr := pb.Peek(4) - if peekErr == nil || (peekErr == io.EOF && len(magic) >= 2) { - switch { - case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: - gzipReader, gzErr := gzip.NewReader(pb) - if gzErr != nil { - if errClose := pb.Close(); errClose != nil { - log.Errorf("response decoder: close body error: %v", errClose) - } - return nil, fmt.Errorf("magic-byte gzip: failed to create reader: %w", gzErr) - } - return &compositeReadCloser{ - Reader: gzipReader, - closers: []func() error{ - gzipReader.Close, - pb.Close, - }, - }, nil - case len(magic) >= 4 && magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd: - decoder, zdErr := zstd.NewReader(pb) - if zdErr != nil { - if errClose := pb.Close(); errClose != nil { - log.Errorf("response decoder: close body error: %v", errClose) - } - return nil, fmt.Errorf("magic-byte zstd: failed to create reader: %w", zdErr) - } - return &compositeReadCloser{ - Reader: decoder, - closers: []func() error{ - func() error { decoder.Close(); return nil }, - pb.Close, - }, - }, nil - } - } - return pb, nil + // Detection waits for the first Read: a streaming upstream may send headers long + // before its first byte, and peeking here would block the caller until then. + return &sniffedResponseBody{body: &peekableBody{Reader: bufio.NewReader(body), closer: body}}, nil } encodings := strings.Split(contentEncoding, ",") reader := io.Reader(body) @@ -133,7 +100,7 @@ func DecodeResponseBody(body io.ReadCloser, header http.Header) (io.ReadCloser, case "br": reader = brotli.NewReader(reader) case "zstd": - decoder, errZstd := zstd.NewReader(reader) + decoder, errZstd := newResponseZstdReader(reader) if errZstd != nil { cleanup() return nil, fmt.Errorf("failed to create zstd reader: %w", errZstd) @@ -179,3 +146,78 @@ func isZlibHeader(header []byte) bool { cmf, flg := header[0], header[1] return cmf&0x0f == 8 && cmf>>4 <= 7 && (uint16(cmf)<<8|uint16(flg))%31 == 0 } + +// newResponseZstdReader decodes one response synchronously. The default decoder hands blocks to up +// to four asynchronous block decoders, which costs allocations and handoff latency and gains +// nothing on bodies that are read as they arrive. +func newResponseZstdReader(reader io.Reader) (*zstd.Decoder, error) { + return zstd.NewReader(reader, zstd.WithDecoderConcurrency(1)) +} + +// sniffedResponseBody decodes a body that arrived without Content-Encoding but starts with +// gzip or zstd magic bytes. Detection runs on the first Read and waits for more than one byte +// only when the first byte can start a gzip or zstd frame. +type sniffedResponseBody struct { + body *peekableBody + once sync.Once + mu sync.Mutex + reader io.Reader + decoder func() error + err error +} + +func (s *sniffedResponseBody) detect() { + reader := io.Reader(s.body) + var closeDecoder func() error + defer func() { + s.mu.Lock() + s.reader, s.decoder = reader, closeDecoder + s.mu.Unlock() + }() + first, _ := s.body.Peek(1) + if len(first) == 0 || (first[0] != 0x1f && first[0] != 0x28) { + return + } + magic, errPeek := s.body.Peek(4) + if errPeek != nil && !(errPeek == io.EOF && len(magic) >= 2) { + return + } + switch { + case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: + gzipReader, errGzip := gzip.NewReader(s.body) + if errGzip != nil { + s.err = fmt.Errorf("magic-byte gzip: failed to create reader: %w", errGzip) + return + } + reader, closeDecoder = gzipReader, gzipReader.Close + case len(magic) >= 4 && magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd: + decoder, errZstd := newResponseZstdReader(s.body) + if errZstd != nil { + s.err = fmt.Errorf("magic-byte zstd: failed to create reader: %w", errZstd) + return + } + reader, closeDecoder = decoder, func() error { decoder.Close(); return nil } + } +} + +func (s *sniffedResponseBody) Read(p []byte) (int, error) { + s.once.Do(s.detect) + if s.err != nil { + return 0, s.err + } + return s.reader.Read(p) +} + +func (s *sniffedResponseBody) Close() error { + s.mu.Lock() + closeDecoder := s.decoder + s.mu.Unlock() + var errDecoder error + if closeDecoder != nil { + errDecoder = closeDecoder() + } + if errClose := s.body.Close(); errClose != nil { + return errClose + } + return errDecoder +} diff --git a/internal/runtime/executor/helps/response_decode_bench_test.go b/internal/runtime/executor/helps/response_decode_bench_test.go new file mode 100644 index 0000000000..f7b5444e9a --- /dev/null +++ b/internal/runtime/executor/helps/response_decode_bench_test.go @@ -0,0 +1,117 @@ +package helps + +import ( + "bytes" + "fmt" + "io" + "math/rand" + "net/http" + "runtime" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" +) + +// benchZstdSSE returns a zstd-encoded body of about size bytes of Responses-style SSE deltas. +func benchZstdSSE(b *testing.B, size int) []byte { + b.Helper() + r := rand.New(rand.NewSource(1)) + words := strings.Fields("the function return error context request response model input output tool call result value index token cache stream event message content type text") + var plain strings.Builder + for i := 0; plain.Len() < size; i++ { + var delta strings.Builder + for j, n := 0, 4+r.Intn(12); j < n; j++ { + delta.WriteString(words[r.Intn(len(words))]) + delta.WriteByte(' ') + } + fmt.Fprintf(&plain, "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"sequence_number\":%d,\"delta\":%q}\n\n", i, delta.String()) + } + encoder, errNewWriter := zstd.NewWriter(nil) + if errNewWriter != nil { + b.Fatal(errNewWriter) + } + defer func() { _ = encoder.Close() }() + return encoder.EncodeAll([]byte(plain.String()), nil) +} + +var benchZstdHeader = http.Header{"Content-Encoding": {"zstd"}} + +// BenchmarkDecodeResponseBodyZstd decodes whole zstd upstream responses concurrently. +func BenchmarkDecodeResponseBodyZstd(b *testing.B) { + for _, size := range []int{64 << 10, 2 << 20} { + wire := benchZstdSSE(b, size) + b.Run(fmt.Sprintf("%dKiB", size>>10), func(b *testing.B) { + b.SetBytes(int64(size)) + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader) + if errDecode != nil { + b.Fatal(errDecode) + } + if _, errCopy := io.Copy(io.Discard, body); errCopy != nil { + b.Fatal(errCopy) + } + _ = body.Close() + } + }) + }) + } +} + +// BenchmarkDecodeResponseBodyOpenStreams holds 256 zstd responses open after their first read, +// like streams waiting on the model, and reports the heap held per open stream. +func BenchmarkDecodeResponseBodyOpenStreams(b *testing.B) { + const streams = 256 + wire := benchZstdSSE(b, 256<<10) + var heap float64 + for i := 0; i < b.N; i++ { + runtime.GC() + var before, during runtime.MemStats + runtime.ReadMemStats(&before) + bodies := make([]io.ReadCloser, streams) + buf := make([]byte, 512) + for s := range bodies { + body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader) + if errDecode != nil { + b.Fatal(errDecode) + } + if _, errRead := body.Read(buf); errRead != nil { + b.Fatal(errRead) + } + bodies[s] = body + } + runtime.GC() + runtime.ReadMemStats(&during) + heap += float64(int64(during.HeapInuse)-int64(before.HeapInuse)) / streams / 1024 + for _, body := range bodies { + _ = body.Close() + } + } + b.ReportMetric(heap/float64(b.N), "heap-KiB/stream") +} + +// BenchmarkDecodeResponseBodyUnlabeledFirstByte measures how long DecodeResponseBody blocks +// for a response without Content-Encoding whose first byte arrives 20 ms after the headers. +func BenchmarkDecodeResponseBodyUnlabeledFirstByte(b *testing.B) { + var blocked time.Duration + for i := 0; i < b.N; i++ { + reader, writer := io.Pipe() + go func() { + time.Sleep(20 * time.Millisecond) + _, _ = writer.Write([]byte("data: {}\n\n")) + _ = writer.Close() + }() + began := time.Now() + body, errDecode := DecodeResponseBody(reader, http.Header{}) + blocked += time.Since(began) + if errDecode != nil { + b.Fatal(errDecode) + } + _, _ = io.Copy(io.Discard, body) + _ = body.Close() + } + b.ReportMetric(float64(blocked.Milliseconds())/float64(b.N), "blocked-ms") +} diff --git a/internal/runtime/executor/helps/response_decode_test.go b/internal/runtime/executor/helps/response_decode_test.go index 999351bad1..bc08169a65 100644 --- a/internal/runtime/executor/helps/response_decode_test.go +++ b/internal/runtime/executor/helps/response_decode_test.go @@ -10,6 +10,7 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" @@ -131,3 +132,56 @@ func TestDecodeResponseBodyCompositeClose(t *testing.T) { t.Fatalf("close must attempt all closers and return first error: %v, order=%v", errClose, order) } } + +func TestDecodeResponseBodyUnlabeledDoesNotWaitForFirstByte(t *testing.T) { + reader, writer := io.Pipe() + returned := make(chan io.ReadCloser, 1) + go func() { + body, errDecode := DecodeResponseBody(reader, http.Header{}) + if errDecode != nil { + t.Error(errDecode) + } + returned <- body + }() + var body io.ReadCloser + select { + case body = <-returned: + case <-time.After(2 * time.Second): + t.Fatal("DecodeResponseBody blocked until the upstream sent its first byte") + } + go func() { + _, _ = writer.Write([]byte("data: {}\n\n")) + _ = writer.Close() + }() + got, errRead := io.ReadAll(body) + if errRead != nil || string(got) != "data: {}\n\n" { + t.Fatalf("read %q, %v", got, errRead) + } + if errClose := body.Close(); errClose != nil { + t.Fatal(errClose) + } +} + +func TestDecodeResponseBodySniffsSplitMagicBytes(t *testing.T) { + plain := []byte("data: response.completed\n\n") + for _, encoding := range []string{"gzip", "zstd"} { + wire := encodeResponseDecodeTestBody(t, encoding, plain) + reader, writer := io.Pipe() + go func() { + // The magic bytes arrive one write at a time. + for i := range wire { + _, _ = writer.Write(wire[i : i+1]) + } + _ = writer.Close() + }() + body, errDecode := DecodeResponseBody(reader, http.Header{}) + if errDecode != nil { + t.Fatal(errDecode) + } + got, errRead := io.ReadAll(body) + if errRead != nil || !bytes.Equal(got, plain) { + t.Fatalf("%s: read %q, %v", encoding, got, errRead) + } + _ = body.Close() + } +}