mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-17 07:08:01 +08:00
DecodeResponseBody used zstd.NewReader with its default options, which hands blocks to up to four asynchronous decoders: allocations and handoff latency on every streamed response for no gain on bodies read as they arrive. Decode with WithDecoderConcurrency(1) instead. A response without Content-Encoding was sniffed for gzip/zstd magic with Peek(4) before DecodeResponseBody returned, blocking the caller until the upstream sent its first bytes. Detect on the first Read instead, waiting for more than one byte only when the first byte can start a gzip or zstd frame.
118 lines
3.6 KiB
Go
118 lines
3.6 KiB
Go
package helps
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// benchZstdSSE returns a zstd-encoded body of about size bytes of Responses-style SSE deltas.
|
|
func benchZstdSSE(b *testing.B, size int) []byte {
|
|
b.Helper()
|
|
r := rand.New(rand.NewSource(1))
|
|
words := strings.Fields("the function return error context request response model input output tool call result value index token cache stream event message content type text")
|
|
var plain strings.Builder
|
|
for i := 0; plain.Len() < size; i++ {
|
|
var delta strings.Builder
|
|
for j, n := 0, 4+r.Intn(12); j < n; j++ {
|
|
delta.WriteString(words[r.Intn(len(words))])
|
|
delta.WriteByte(' ')
|
|
}
|
|
fmt.Fprintf(&plain, "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"sequence_number\":%d,\"delta\":%q}\n\n", i, delta.String())
|
|
}
|
|
encoder, errNewWriter := zstd.NewWriter(nil)
|
|
if errNewWriter != nil {
|
|
b.Fatal(errNewWriter)
|
|
}
|
|
defer func() { _ = encoder.Close() }()
|
|
return encoder.EncodeAll([]byte(plain.String()), nil)
|
|
}
|
|
|
|
var benchZstdHeader = http.Header{"Content-Encoding": {"zstd"}}
|
|
|
|
// BenchmarkDecodeResponseBodyZstd decodes whole zstd upstream responses concurrently.
|
|
func BenchmarkDecodeResponseBodyZstd(b *testing.B) {
|
|
for _, size := range []int{64 << 10, 2 << 20} {
|
|
wire := benchZstdSSE(b, size)
|
|
b.Run(fmt.Sprintf("%dKiB", size>>10), func(b *testing.B) {
|
|
b.SetBytes(int64(size))
|
|
b.ReportAllocs()
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
for pb.Next() {
|
|
body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader)
|
|
if errDecode != nil {
|
|
b.Fatal(errDecode)
|
|
}
|
|
if _, errCopy := io.Copy(io.Discard, body); errCopy != nil {
|
|
b.Fatal(errCopy)
|
|
}
|
|
_ = body.Close()
|
|
}
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
// BenchmarkDecodeResponseBodyOpenStreams holds 256 zstd responses open after their first read,
|
|
// like streams waiting on the model, and reports the heap held per open stream.
|
|
func BenchmarkDecodeResponseBodyOpenStreams(b *testing.B) {
|
|
const streams = 256
|
|
wire := benchZstdSSE(b, 256<<10)
|
|
var heap float64
|
|
for i := 0; i < b.N; i++ {
|
|
runtime.GC()
|
|
var before, during runtime.MemStats
|
|
runtime.ReadMemStats(&before)
|
|
bodies := make([]io.ReadCloser, streams)
|
|
buf := make([]byte, 512)
|
|
for s := range bodies {
|
|
body, errDecode := DecodeResponseBody(io.NopCloser(bytes.NewReader(wire)), benchZstdHeader)
|
|
if errDecode != nil {
|
|
b.Fatal(errDecode)
|
|
}
|
|
if _, errRead := body.Read(buf); errRead != nil {
|
|
b.Fatal(errRead)
|
|
}
|
|
bodies[s] = body
|
|
}
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&during)
|
|
heap += float64(int64(during.HeapInuse)-int64(before.HeapInuse)) / streams / 1024
|
|
for _, body := range bodies {
|
|
_ = body.Close()
|
|
}
|
|
}
|
|
b.ReportMetric(heap/float64(b.N), "heap-KiB/stream")
|
|
}
|
|
|
|
// BenchmarkDecodeResponseBodyUnlabeledFirstByte measures how long DecodeResponseBody blocks
|
|
// for a response without Content-Encoding whose first byte arrives 20 ms after the headers.
|
|
func BenchmarkDecodeResponseBodyUnlabeledFirstByte(b *testing.B) {
|
|
var blocked time.Duration
|
|
for i := 0; i < b.N; i++ {
|
|
reader, writer := io.Pipe()
|
|
go func() {
|
|
time.Sleep(20 * time.Millisecond)
|
|
_, _ = writer.Write([]byte("data: {}\n\n"))
|
|
_ = writer.Close()
|
|
}()
|
|
began := time.Now()
|
|
body, errDecode := DecodeResponseBody(reader, http.Header{})
|
|
blocked += time.Since(began)
|
|
if errDecode != nil {
|
|
b.Fatal(errDecode)
|
|
}
|
|
_, _ = io.Copy(io.Discard, body)
|
|
_ = body.Close()
|
|
}
|
|
b.ReportMetric(float64(blocked.Milliseconds())/float64(b.N), "blocked-ms")
|
|
}
|