diff --git a/internal/runtime/executor/devin_executor.go b/internal/runtime/executor/devin_executor.go index bc11b858c..88b88313c 100644 --- a/internal/runtime/executor/devin_executor.go +++ b/internal/runtime/executor/devin_executor.go @@ -569,6 +569,14 @@ func (e *DevinExecutor) streamDevinFrames( if frameRes.Usage != nil { finalUsage = frameRes.Usage + } else if len(frameRes.ResponseDimensionGroups) > 0 && finalUsage == nil { + if inTok, outTok, cachedTok, ok := helps.ParseDevinResponseDimensionGroups(frameRes.ResponseDimensionGroups); ok { + finalUsage = &helps.DevinUsage{ + PromptTokens: inTok, + CompletionTokens: outTok, + CachedTokens: cachedTok, + } + } } if len(frameRes.DeltaSignature) > 0 { accumulatedSignature = append(accumulatedSignature, frameRes.DeltaSignature...) @@ -906,6 +914,14 @@ func consumeDevinFramesToInteractions(body io.Reader, model, chatModelUID string if frameRes.Usage != nil { finalUsage = frameRes.Usage + } else if len(frameRes.ResponseDimensionGroups) > 0 && finalUsage == nil { + if inTok, outTok, cachedTok, ok := helps.ParseDevinResponseDimensionGroups(frameRes.ResponseDimensionGroups); ok { + finalUsage = &helps.DevinUsage{ + PromptTokens: inTok, + CompletionTokens: outTok, + CachedTokens: cachedTok, + } + } } if len(frameRes.DeltaSignature) > 0 { accumulatedSignature = append(accumulatedSignature, frameRes.DeltaSignature...) diff --git a/internal/runtime/executor/helps/devin_wire.go b/internal/runtime/executor/helps/devin_wire.go index 1f214e05b..79b96ab03 100644 --- a/internal/runtime/executor/helps/devin_wire.go +++ b/internal/runtime/executor/helps/devin_wire.go @@ -94,12 +94,13 @@ type DevinPrompt struct { // DevinUsage captures token accounting from response Field 7. type DevinUsage struct { - PromptTokens int64 - CompletionTokens int64 - CachedTokens int64 - StatusCode uint64 - RequestID string - ModelName string + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CachedTokens int64 `json:"cached_tokens"` + StatusCode uint64 `json:"status_code,omitempty"` + RequestID string `json:"request_id,omitempty"` + ModelName string `json:"model_name,omitempty"` + Headers map[string]string `json:"headers,omitempty"` } // DevinFrameResult represents decoded content from a single Connect-proto frame. @@ -715,6 +716,42 @@ func parseDevinTimestamp(data []byte) uint64 { return secs } +// parseDevinHeaderField parses a repeated submessage in Field 7 (subfield 8) representing upstream response headers: +// Tag 1 (string): Header name (e.g. "x-request-id", "Request-Id", "openai-processing-ms") +// Tag 2 (string): Header value (e.g. "req_011Cf1JivhJrXDq9ycq7cEtH", "chatcmpl-...") +func parseDevinHeaderField(data []byte) (string, string) { + var key, val string + pos := 0 + for pos < len(data) { + num, typ, n := protowire.ConsumeTag(data[pos:]) + if n <= 0 { + break + } + pos += n + switch typ { + case protowire.BytesType: + b, bn := protowire.ConsumeBytes(data[pos:]) + if bn <= 0 { + return key, val + } + pos += bn + switch num { + case 1: + key = string(b) + case 2: + val = string(b) + } + default: + nSkip := protowire.ConsumeFieldValue(num, typ, data[pos:]) + if nSkip <= 0 { + return key, val + } + pos += nSkip + } + } + return key, val +} + func parseDevinUsageField(data []byte) *DevinUsage { u := &DevinUsage{} pos := 0 @@ -733,10 +770,12 @@ func parseDevinUsageField(data []byte) *DevinUsage { } pos += vn switch num { - case 2: // Prompt tokens (uncached input) - u.PromptTokens = int64(v) + case 2: // Prompt tokens (uncached input from turn message) + u.PromptTokens += int64(v) case 3: // Output tokens u.CompletionTokens = int64(v) + case 4: // Additional context/system prompt tokens in OpenAI-family models (total prompt = 2 + 4) + u.PromptTokens += int64(v) case 5: // Cache read tokens u.CachedTokens = int64(v) case 6: // Status code @@ -750,7 +789,18 @@ func parseDevinUsageField(data []byte) *DevinUsage { pos += bn switch num { case 8: - u.RequestID = string(val) + k, v := parseDevinHeaderField(val) + if k != "" { + if u.Headers == nil { + u.Headers = make(map[string]string) + } + u.Headers[k] = v + if strings.EqualFold(k, "x-request-id") || strings.EqualFold(k, "request-id") { + u.RequestID = v + } + } else if isPrintableASCII(val) { + u.RequestID = string(val) + } case 9: u.ModelName = string(val) } @@ -767,12 +817,148 @@ func parseDevinUsageField(data []byte) *DevinUsage { } pos += fn default: - return u + nSkip := protowire.ConsumeFieldValue(num, typ, data[pos:]) + if nSkip <= 0 { + return u + } + pos += nSkip } } return u } +// ParseDevinResponseDimensionGroups parses Field 28 (ResponseDimensionGroups) to extract Token Usage metrics: +// input_tokens, output_tokens, cached_input_tokens. +func ParseDevinResponseDimensionGroups(data []byte) (promptTokens, completionTokens, cachedTokens int64, found bool) { + pos := 0 + for pos < len(data) { + num, typ, n := protowire.ConsumeTag(data[pos:]) + if n <= 0 { + break + } + pos += n + if typ != protowire.BytesType { + nSkip := protowire.ConsumeFieldValue(num, typ, data[pos:]) + if nSkip <= 0 { + break + } + pos += nSkip + continue + } + gBytes, bn := protowire.ConsumeBytes(data[pos:]) + if bn <= 0 { + break + } + pos += bn + if num != 28 { + continue + } + + gPos := 0 + var title string + type metricItem struct { + key string + val float32 + } + var metrics []metricItem + for gPos < len(gBytes) { + gNum, gTyp, gn := protowire.ConsumeTag(gBytes[gPos:]) + if gn <= 0 { + break + } + gPos += gn + if gTyp != protowire.BytesType { + gSkip := protowire.ConsumeFieldValue(gNum, gTyp, gBytes[gPos:]) + if gSkip <= 0 { + break + } + gPos += gSkip + continue + } + gb, gbn := protowire.ConsumeBytes(gBytes[gPos:]) + if gbn <= 0 { + break + } + gPos += gbn + if gNum == 1 { + title = string(gb) + } else if gNum == 2 { + mPos := 0 + var mKey string + var mVal float32 + for mPos < len(gb) { + mNum, mTyp, mn := protowire.ConsumeTag(gb[mPos:]) + if mn <= 0 { + break + } + mPos += mn + if mTyp != protowire.BytesType { + mSkip := protowire.ConsumeFieldValue(mNum, mTyp, gb[mPos:]) + if mSkip <= 0 { + break + } + mPos += mSkip + continue + } + mb, mbn := protowire.ConsumeBytes(gb[mPos:]) + if mbn <= 0 { + break + } + mPos += mbn + if mNum == 5 { + mKey = string(mb) + } else if mNum == 4 { + dPos := 0 + for dPos < len(mb) { + dNum, dTyp, dn := protowire.ConsumeTag(mb[dPos:]) + if dn <= 0 { + break + } + dPos += dn + if dTyp == protowire.Fixed32Type { + dv, dfn := protowire.ConsumeFixed32(mb[dPos:]) + if dfn <= 0 { + break + } + dPos += dfn + if dNum == 2 { + mVal = math.Float32frombits(dv) + } + } else { + dSkip := protowire.ConsumeFieldValue(dNum, dTyp, mb[dPos:]) + if dSkip <= 0 { + break + } + dPos += dSkip + } + } + } + } + if mKey != "" { + metrics = append(metrics, metricItem{key: mKey, val: mVal}) + } + } + } + + if strings.EqualFold(title, "Token Usage") { + for _, m := range metrics { + switch m.key { + case "input_tokens": + promptTokens = int64(m.val) + found = true + case "output_tokens": + completionTokens = int64(m.val) + found = true + case "cached_input_tokens": + cachedTokens = int64(m.val) + found = true + } + } + } + } + return promptTokens, completionTokens, cachedTokens, found +} + // ParseDevinTrailerError inspects Connect-RPC EOS trailer frames and maps error status codes. func ParseDevinTrailerError(payload []byte) (statusCode int, err error) { trimmed := bytes.TrimSpace(payload) diff --git a/internal/runtime/executor/helps/devin_wire_test.go b/internal/runtime/executor/helps/devin_wire_test.go index 3255c5e17..71d924336 100644 --- a/internal/runtime/executor/helps/devin_wire_test.go +++ b/internal/runtime/executor/helps/devin_wire_test.go @@ -588,3 +588,120 @@ func extractField15Subfields(t *testing.T, reqBytes []byte) (string, map[int]uin } return sessionID, subfields } + +func TestParseDevinUsageField_HeadersAndField4(t *testing.T) { + var f7Bytes []byte + f7Bytes = protowire.AppendTag(f7Bytes, 2, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 3) + + f7Bytes = protowire.AppendTag(f7Bytes, 4, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 58) + + f7Bytes = protowire.AppendTag(f7Bytes, 3, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 39) + + f7Bytes = protowire.AppendTag(f7Bytes, 5, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 19179) + + f7Bytes = protowire.AppendTag(f7Bytes, 6, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 66) + + // Submessage 1: openai-version + var h1 []byte + h1 = protowire.AppendTag(h1, 1, protowire.BytesType) + h1 = protowire.AppendString(h1, "openai-version") + h1 = protowire.AppendTag(h1, 2, protowire.BytesType) + h1 = protowire.AppendString(h1, "2020-10-01") + f7Bytes = protowire.AppendTag(f7Bytes, 8, protowire.BytesType) + f7Bytes = protowire.AppendBytes(f7Bytes, h1) + + // Submessage 2: x-request-id + var h2 []byte + h2 = protowire.AppendTag(h2, 1, protowire.BytesType) + h2 = protowire.AppendString(h2, "x-request-id") + h2 = protowire.AppendTag(h2, 2, protowire.BytesType) + h2 = protowire.AppendString(h2, "req_5bb00ad48ae048119e3420bddf36257f") + f7Bytes = protowire.AppendTag(f7Bytes, 8, protowire.BytesType) + f7Bytes = protowire.AppendBytes(f7Bytes, h2) + + // Submessage 3: openai-processing-ms + var h3 []byte + h3 = protowire.AppendTag(h3, 1, protowire.BytesType) + h3 = protowire.AppendString(h3, "openai-processing-ms") + h3 = protowire.AppendTag(h3, 2, protowire.BytesType) + h3 = protowire.AppendString(h3, "419") + f7Bytes = protowire.AppendTag(f7Bytes, 8, protowire.BytesType) + f7Bytes = protowire.AppendBytes(f7Bytes, h3) + + f7Bytes = protowire.AppendTag(f7Bytes, 9, protowire.BytesType) + f7Bytes = protowire.AppendString(f7Bytes, "gpt-5-6-luna-low") + + usage := parseDevinUsageField(f7Bytes) + if usage == nil { + t.Fatal("expected non-nil usage") + } + + // 3 + 58 = 61 + if usage.PromptTokens != 61 { + t.Errorf("PromptTokens = %d, want 61 (3 turn + 58 context)", usage.PromptTokens) + } + if usage.CompletionTokens != 39 { + t.Errorf("CompletionTokens = %d, want 39", usage.CompletionTokens) + } + if usage.CachedTokens != 19179 { + t.Errorf("CachedTokens = %d, want 19179", usage.CachedTokens) + } + if usage.StatusCode != 66 { + t.Errorf("StatusCode = %d, want 66", usage.StatusCode) + } + if usage.RequestID != "req_5bb00ad48ae048119e3420bddf36257f" { + t.Errorf("RequestID = %q, want clean request-id", usage.RequestID) + } + if usage.ModelName != "gpt-5-6-luna-low" { + t.Errorf("ModelName = %q, want gpt-5-6-luna-low", usage.ModelName) + } + if usage.Headers["openai-processing-ms"] != "419" { + t.Errorf("header processing-ms = %q, want 419", usage.Headers["openai-processing-ms"]) + } + if usage.Headers["openai-version"] != "2020-10-01" { + t.Errorf("header openai-version = %q, want 2020-10-01", usage.Headers["openai-version"]) + } +} + +func TestParseDevinUsageField_AnthropicRequestId(t *testing.T) { + var f7Bytes []byte + f7Bytes = protowire.AppendTag(f7Bytes, 2, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 4) + + f7Bytes = protowire.AppendTag(f7Bytes, 3, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 109) + + f7Bytes = protowire.AppendTag(f7Bytes, 5, protowire.VarintType) + f7Bytes = protowire.AppendVarint(f7Bytes, 577) + + // Anthropic uses capitalized "Request-Id" + var h []byte + h = protowire.AppendTag(h, 1, protowire.BytesType) + h = protowire.AppendString(h, "Request-Id") + h = protowire.AppendTag(h, 2, protowire.BytesType) + h = protowire.AppendString(h, "req_011Cf1JivhJrXDq9ycq7cEtH") + f7Bytes = protowire.AppendTag(f7Bytes, 8, protowire.BytesType) + f7Bytes = protowire.AppendBytes(f7Bytes, h) + + usage := parseDevinUsageField(f7Bytes) + if usage == nil { + t.Fatal("expected non-nil usage") + } + if usage.RequestID != "req_011Cf1JivhJrXDq9ycq7cEtH" { + t.Errorf("RequestID = %q, want req_011Cf1JivhJrXDq9ycq7cEtH", usage.RequestID) + } + if usage.PromptTokens != 4 { + t.Errorf("PromptTokens = %d, want 4", usage.PromptTokens) + } + if usage.CompletionTokens != 109 { + t.Errorf("CompletionTokens = %d, want 109", usage.CompletionTokens) + } + if usage.CachedTokens != 577 { + t.Errorf("CachedTokens = %d, want 577", usage.CachedTokens) + } +}