mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-16 22:57:53 +08:00
Add attempt-scoped HomeSelection metadata carrying the reused and additional flags, scheduling route model, and credential policy through usage records to the Redis queue. Mark retained executions and Codex duplex successors as reused so Home can compensate later turns after completion without redispatching or reacquiring a concurrency slot. Mark split additional-model records as additional so they never add scheduling charges, while preserving independent per-response execution IDs for deduplication. Document the deferred accounting and add coverage across conductor, duplex, reporter, and queue.
918 lines
34 KiB
Go
918 lines
34 KiB
Go
package executor
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/MoeForge/moego/ojson"
|
|
"github.com/gorilla/websocket"
|
|
"github.com/router-for-me/CLIProxyAPI/v8/internal/runtime/executor/helps"
|
|
"github.com/router-for-me/CLIProxyAPI/v8/internal/thinking"
|
|
cliproxyauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
|
|
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
|
coreusage "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage"
|
|
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// codexDuplexConnectionError keeps a connection failure from cooling the shared
|
|
// credential. The socket still fails visibly; no reconnect or replay is attempted.
|
|
// Auth/quota failures from the initial handshake retain the normal policy.
|
|
type codexDuplexConnectionError struct{ cause error }
|
|
|
|
func (e *codexDuplexConnectionError) Error() string { return e.cause.Error() }
|
|
func (e *codexDuplexConnectionError) Unwrap() error { return e.cause }
|
|
func (e *codexDuplexConnectionError) IsRequestScoped() bool { return true }
|
|
|
|
// streamCodexDuplex owns the already authenticated socket until downstream
|
|
// disconnect. A response terminal event is not a connection terminal event:
|
|
// accepted steering may produce a successor or wait for client tool results.
|
|
// There is no redial, credential selection, local acknowledgement, or replay
|
|
// in this path. Only the upstream can take ownership of a steering submission.
|
|
func (e *CodexWebsocketsExecutor) streamCodexDuplex(
|
|
ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options,
|
|
sess *codexWebsocketSession, conn *websocket.Conn, readCh chan codexWebsocketRead,
|
|
input <-chan cliproxyexecutor.WebsocketInput, initial *codexWebsocketPrepared,
|
|
initialReporter *helps.UsageReporter, headers http.Header, unlock func(),
|
|
) *cliproxyexecutor.StreamResult {
|
|
streamCtx, cancel := context.WithCancel(ctx)
|
|
// Duplex successors bypass dispatch, including automatic steering responses.
|
|
followupCtx := ctx
|
|
if selection := coreusage.HomeSelectionFromContext(ctx); selection != nil {
|
|
selection.Reused = true
|
|
followupCtx = coreusage.WithHomeSelection(ctx, *selection)
|
|
}
|
|
out := make(chan cliproxyexecutor.StreamChunk)
|
|
// This first frame was successfully written by ExecuteStream before handoff.
|
|
log.Infof("codex websockets: request forwarded session=%s auth=%s url=%s event=response.create", sess.sessionID, auth.ID, initial.wsURL)
|
|
writeErrors := make(chan error, 1)
|
|
// Explicit creates have their own request settings. Automatic successors
|
|
// inherit the preceding response settings, just as they do upstream.
|
|
var metadataMu sync.Mutex
|
|
pending := []*codexWebsocketPrepared{initial}
|
|
// Serialize explicit creates against steering continuations. A response.created
|
|
// alone does not identify whether upstream created it automatically.
|
|
type codexSteerSubmission struct{ parent, eventID string }
|
|
// successorID is set once the response that consumes the steer has started; the steer
|
|
// stays outstanding until that response ends.
|
|
type codexAcceptedSteer struct{ parent, eventID, successorID string }
|
|
var unacknowledgedSteers []codexSteerSubmission
|
|
acceptedSteers := make(map[string]codexAcceptedSteer)
|
|
current := initial
|
|
responseID := ""
|
|
responseSettings := make(map[string]*codexWebsocketPrepared)
|
|
steeringSettings := make(map[string]*codexWebsocketPrepared)
|
|
var responseOrder []string
|
|
releaseSteeringSettings := func(parent string) {
|
|
for _, target := range unacknowledgedSteers {
|
|
if target.parent == parent {
|
|
return
|
|
}
|
|
}
|
|
for _, target := range acceptedSteers {
|
|
if target.parent == parent {
|
|
return
|
|
}
|
|
}
|
|
delete(steeringSettings, parent)
|
|
}
|
|
removeAcceptedSteer := func(id string) {
|
|
parent := acceptedSteers[id].parent
|
|
delete(acceptedSteers, id)
|
|
releaseSteeringSettings(parent)
|
|
}
|
|
waitingParent := ""
|
|
automaticActive := false
|
|
stateChanged := make(chan struct{}, 1)
|
|
wakeWriter := func() {
|
|
select {
|
|
case stateChanged <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
waitFor := func(ready func() bool) bool {
|
|
for {
|
|
metadataMu.Lock()
|
|
ok := ready()
|
|
metadataMu.Unlock()
|
|
if ok {
|
|
return streamCtx.Err() == nil
|
|
}
|
|
select {
|
|
case <-stateChanged:
|
|
case <-streamCtx.Done():
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
// Do not consume follow-ups until bootstrap succeeds. A rejected initial
|
|
// request may retry on another credential using the same input channel.
|
|
inputReady := make(chan struct{})
|
|
writerDone := make(chan struct{})
|
|
var pendingCreates [][]byte
|
|
// Only the writer mutates this; the loss path reads it after joining writerDone.
|
|
// A message the writer took from input but neither sent nor queued was never written.
|
|
var consumedPayload []byte
|
|
readyForCreate := func() bool {
|
|
metadataMu.Lock()
|
|
defer metadataMu.Unlock()
|
|
if len(unacknowledgedSteers) > 0 || automaticActive {
|
|
return false
|
|
}
|
|
for _, accepted := range acceptedSteers {
|
|
if accepted.successorID == "" && accepted.parent != waitingParent {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
go func() {
|
|
defer close(writerDone)
|
|
fail := func(err error) {
|
|
select {
|
|
case writeErrors <- err:
|
|
default:
|
|
}
|
|
cancel()
|
|
}
|
|
reject := func(message string) bool {
|
|
payload, _ := json.Marshal(map[string]any{
|
|
"type": "error", "status": http.StatusBadRequest,
|
|
"error": map[string]string{"type": "invalid_request_error", "message": message},
|
|
})
|
|
// Reader cleanup joins this writer before closing out. Cancellation must
|
|
// also release a local error blocked behind a disconnected downstream.
|
|
select {
|
|
case out <- cliproxyexecutor.StreamChunk{Payload: payload}:
|
|
consumedPayload = nil
|
|
return true
|
|
case <-streamCtx.Done():
|
|
return false
|
|
}
|
|
}
|
|
initialModel := ""
|
|
if initial != nil {
|
|
initialModel = strings.TrimSpace(ojson.GetBytesNoCopy(initial.originalPayload, "model").String())
|
|
}
|
|
boundModel := helps.PayloadRequestedModel(opts, initialModel)
|
|
if boundModel == "" {
|
|
boundModel = req.Model
|
|
}
|
|
authorizePayload := func(payload []byte, forceBoundModel bool) (allowed bool, keepOpen bool) {
|
|
authPayload := payload
|
|
explicitModel := strings.TrimSpace(ojson.GetBytesNoCopy(authPayload, "model").String())
|
|
if boundModel != "" && json.Valid(authPayload) && (forceBoundModel || explicitModel == "" || explicitModel == req.Model) {
|
|
if withModel, errSet := ojson.SetBytes(bytes.Clone(authPayload), "model", boundModel); errSet == nil {
|
|
authPayload = withModel
|
|
}
|
|
}
|
|
errPayload, errAuth := cliproxyexecutor.AuthorizeWebsocketTurn(streamCtx, authPayload)
|
|
if errAuth != nil {
|
|
fail(errAuth)
|
|
return false, false
|
|
}
|
|
if len(errPayload) > 0 {
|
|
select {
|
|
case out <- cliproxyexecutor.StreamChunk{Payload: errPayload}:
|
|
consumedPayload = nil
|
|
return false, true
|
|
case <-streamCtx.Done():
|
|
return false, false
|
|
}
|
|
}
|
|
return true, true
|
|
}
|
|
finalBeforeHeaders := func(current, intercepted http.Header, clear []string) http.Header {
|
|
if intercepted != nil {
|
|
if len(intercepted) == 0 {
|
|
return nil
|
|
}
|
|
return intercepted.Clone()
|
|
}
|
|
if len(current) == 0 {
|
|
return nil
|
|
}
|
|
out := current.Clone()
|
|
for _, key := range clear {
|
|
out.Del(key)
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
mergeAfterHeaders := func(current, updates http.Header, clear []string) http.Header {
|
|
if updates == nil && len(clear) == 0 {
|
|
if len(current) == 0 {
|
|
return nil
|
|
}
|
|
return current.Clone()
|
|
}
|
|
out := http.Header{}
|
|
if len(current) > 0 {
|
|
out = current.Clone()
|
|
}
|
|
for _, key := range clear {
|
|
out.Del(key)
|
|
}
|
|
for key, values := range updates {
|
|
out.Del(key)
|
|
for _, value := range values {
|
|
out.Add(key, value)
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
rejectTerminated := func(resp cliproxyexecutor.RequestAfterAuthInterceptResponse) bool {
|
|
status := resp.StatusCode
|
|
if status < 400 || status > 599 {
|
|
status = http.StatusForbidden
|
|
}
|
|
msg := "request terminated by plugin"
|
|
if len(resp.ResponseBody) > 0 {
|
|
if parsed := strings.TrimSpace(ojson.GetBytesNoCopy(resp.ResponseBody, "error.message").String()); parsed != "" {
|
|
msg = parsed
|
|
} else if trimmed := strings.TrimSpace(string(resp.ResponseBody)); trimmed != "" {
|
|
msg = trimmed
|
|
}
|
|
}
|
|
payload, _ := json.Marshal(map[string]any{
|
|
"type": "error", "status": status,
|
|
"error": map[string]string{"type": "invalid_request_error", "message": msg},
|
|
})
|
|
select {
|
|
case out <- cliproxyexecutor.StreamChunk{Payload: payload}:
|
|
consumedPayload = nil
|
|
return true
|
|
case <-streamCtx.Done():
|
|
return false
|
|
}
|
|
}
|
|
interceptTurnPayload := func(payload []byte, baseHeaders http.Header) ([]byte, http.Header, bool, bool) {
|
|
if opts.RequestBeforeAuthInterceptor == nil && opts.RequestAfterAuthInterceptor == nil {
|
|
return payload, baseHeaders, true, true
|
|
}
|
|
curHeaders := baseHeaders
|
|
if curHeaders != nil {
|
|
curHeaders = curHeaders.Clone()
|
|
}
|
|
interceptReq := cliproxyexecutor.RequestAfterAuthInterceptRequest{
|
|
SourceFormat: opts.SourceFormat,
|
|
Model: req.Model,
|
|
RequestedModel: boundModel,
|
|
Stream: true,
|
|
Headers: curHeaders,
|
|
Body: payload,
|
|
Metadata: opts.Metadata,
|
|
}
|
|
if opts.RequestBeforeAuthInterceptor != nil {
|
|
resp := opts.RequestBeforeAuthInterceptor(streamCtx, interceptReq)
|
|
curHeaders = finalBeforeHeaders(curHeaders, resp.Headers, resp.ClearHeaders)
|
|
if len(resp.Body) > 0 {
|
|
payload = resp.Body
|
|
interceptReq.Body = payload
|
|
}
|
|
interceptReq.Headers = curHeaders
|
|
if resp.Terminate {
|
|
return payload, curHeaders, false, rejectTerminated(resp)
|
|
}
|
|
}
|
|
if opts.RequestAfterAuthInterceptor != nil {
|
|
interceptReq.ToFormat = sdktranslator.FromString("codex")
|
|
resp := opts.RequestAfterAuthInterceptor(streamCtx, interceptReq)
|
|
curHeaders = mergeAfterHeaders(curHeaders, resp.Headers, resp.ClearHeaders)
|
|
if len(resp.Body) > 0 {
|
|
payload = resp.Body
|
|
}
|
|
if resp.Terminate {
|
|
return payload, curHeaders, false, rejectTerminated(resp)
|
|
}
|
|
}
|
|
return payload, curHeaders, true, true
|
|
}
|
|
processCreatePayload := func(payload []byte) bool {
|
|
consumedPayload = payload
|
|
if allowed, keepOpen := authorizePayload(payload, false); !allowed {
|
|
return keepOpen
|
|
}
|
|
metadataMu.Lock()
|
|
isAppend := ojson.GetBytesNoCopy(payload, "type").String() == "response.append"
|
|
prevID := strings.TrimSpace(ojson.GetBytesNoCopy(payload, "previous_response_id").String())
|
|
if isAppend && prevID == "" && responseID != "" {
|
|
prevID = responseID
|
|
payload, _ = ojson.SetBytes(payload, "previous_response_id", prevID)
|
|
}
|
|
wrongParent := false
|
|
for _, accepted := range acceptedSteers {
|
|
if accepted.successorID == "" && prevID != waitingParent {
|
|
wrongParent = true
|
|
}
|
|
}
|
|
metadataMu.Unlock()
|
|
if wrongParent {
|
|
if !reject("response.create must continue the response waiting for required input") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
model := strings.TrimSpace(ojson.GetBytesNoCopy(payload, "model").String())
|
|
if model != "" && model != req.Model && model != boundModel {
|
|
fail(cliproxyexecutor.NewUpstreamWebsocketReplayRequiredError())
|
|
return false
|
|
}
|
|
var turnHeaders http.Header
|
|
var allowed, keepOpen bool
|
|
payload, turnHeaders, allowed, keepOpen = interceptTurnPayload(payload, opts.Headers)
|
|
if !allowed {
|
|
return keepOpen
|
|
}
|
|
if model == "" {
|
|
model = req.Model
|
|
if model == "" {
|
|
model = boundModel
|
|
}
|
|
}
|
|
createDoc := ojson.NewDoc(payload)
|
|
_ = createDoc.Set("model", model)
|
|
if isAppend && !createDoc.Get("instructions").Exists() {
|
|
metadataMu.Lock()
|
|
parentSettings := responseSettings[prevID]
|
|
metadataMu.Unlock()
|
|
targetSettings := parentSettings
|
|
if targetSettings == nil {
|
|
targetSettings = initial
|
|
}
|
|
var inst ojson.Result
|
|
if targetSettings != nil {
|
|
inst = ojson.GetBytesNoCopy(targetSettings.clientBody, "instructions")
|
|
if !inst.Exists() {
|
|
inst = ojson.GetBytesNoCopy(targetSettings.originalPayload, "instructions")
|
|
}
|
|
}
|
|
if !inst.Exists() && initial != nil {
|
|
inst = ojson.GetBytesNoCopy(initial.clientBody, "instructions")
|
|
if !inst.Exists() {
|
|
inst = ojson.GetBytesNoCopy(initial.originalPayload, "instructions")
|
|
}
|
|
}
|
|
if inst.Exists() {
|
|
_ = createDoc.SetRawBytes("instructions", []byte(inst.Raw))
|
|
}
|
|
}
|
|
payload = createDoc.Bytes()
|
|
nextReq, nextOpts := req, opts
|
|
nextReq.Payload = payload
|
|
nextOpts.OriginalRequest = payload
|
|
nextOpts.Headers = turnHeaders
|
|
prepared, errPrepare := e.prepareCodexWebsocketStream(streamCtx, auth, nextReq, nextOpts)
|
|
if errPrepare != nil {
|
|
fail(errPrepare)
|
|
return false
|
|
}
|
|
if prepared.wsURL != initial.wsURL {
|
|
fail(cliproxyexecutor.NewUpstreamWebsocketReplayRequiredError())
|
|
return false
|
|
}
|
|
payload = frameCodexWebsocketRequestBody(prepared.clientBody)
|
|
metadataMu.Lock()
|
|
if len(pending) >= 16 {
|
|
metadataMu.Unlock()
|
|
fail(fmt.Errorf("too many outstanding response.create requests"))
|
|
return false
|
|
}
|
|
pending = append(pending, prepared)
|
|
consumedPayload = nil // pending owns it now, including a possibly partial write
|
|
metadataMu.Unlock()
|
|
if prepared.optimizeMultiAgentV2 || prepared.multiAgentV2Conflict {
|
|
sess.setMultiAgentV2Optimized(conn, prepared.optimizeMultiAgentV2 && !prepared.multiAgentV2Conflict)
|
|
}
|
|
if !cliproxyexecutor.WebsocketAuthEnabled(streamCtx, auth.ID) {
|
|
fail(fmt.Errorf("websocket credential is no longer enabled"))
|
|
return false
|
|
}
|
|
helps.RecordAPIWebsocketRequest(streamCtx, e.cfg, helps.UpstreamRequestLog{
|
|
URL: initial.wsURL, Method: "WEBSOCKET", Body: payload, Provider: e.Identifier(), AuthID: auth.ID,
|
|
})
|
|
if errWrite := writeCodexWebsocketMessage(sess, conn, payload); errWrite != nil {
|
|
fail(mapCodexWebsocketWriteError(sess, conn, errWrite))
|
|
return false
|
|
}
|
|
log.Infof("codex websockets: request forwarded session=%s auth=%s url=%s event=%s", sess.sessionID, auth.ID, initial.wsURL, ojson.GetBytesNoCopy(payload, "type").String())
|
|
return true
|
|
}
|
|
flushPendingCreates := func() bool {
|
|
for len(pendingCreates) > 0 && readyForCreate() {
|
|
item := pendingCreates[0]
|
|
pendingCreates = pendingCreates[1:]
|
|
consumedPayload = item
|
|
if !processCreatePayload(item) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
select {
|
|
case <-streamCtx.Done():
|
|
return
|
|
case <-inputReady:
|
|
}
|
|
for {
|
|
if !flushPendingCreates() {
|
|
return
|
|
}
|
|
select {
|
|
case <-streamCtx.Done():
|
|
return
|
|
case <-stateChanged:
|
|
if !flushPendingCreates() {
|
|
return
|
|
}
|
|
continue
|
|
case message, ok := <-input:
|
|
if !ok {
|
|
fail(context.Canceled)
|
|
return
|
|
}
|
|
if message.Err != nil {
|
|
fail(message.Err)
|
|
return
|
|
}
|
|
if !cliproxyexecutor.WebsocketAuthEnabled(streamCtx, auth.ID) {
|
|
// A fresh client connection can select an enabled credential. Do not send
|
|
// this frame on a disabled account or replay it on a different socket.
|
|
fail(fmt.Errorf("websocket credential is no longer enabled"))
|
|
return
|
|
}
|
|
payload := message.Payload
|
|
consumedPayload = payload
|
|
if !json.Valid(payload) {
|
|
if !reject("invalid websocket request JSON") {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
switch ojson.GetBytesNoCopy(payload, "type").String() {
|
|
case "response.steer":
|
|
if allowed, keepOpen := authorizePayload(payload, true); !allowed {
|
|
if !keepOpen {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
var steerHeaders http.Header
|
|
var allowed, keepOpen bool
|
|
payload, steerHeaders, allowed, keepOpen = interceptTurnPayload(payload, opts.Headers)
|
|
if !allowed {
|
|
if !keepOpen {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
// Steering carries business input but must not inherit response.create defaults.
|
|
steerReq := req
|
|
steerReq.Payload = payload
|
|
steerOpts := opts
|
|
steerOpts.OriginalRequest = payload
|
|
steerOpts.Headers = steerHeaders
|
|
if errExpand := helps.ExpandRecognizedResponsesCompactionCapsules(&steerReq, &steerOpts); errExpand != nil {
|
|
fail(statusErr{code: http.StatusBadRequest, msg: errExpand.Error()})
|
|
return
|
|
}
|
|
steerDoc := ojson.NewDoc(steerReq.Payload)
|
|
if codexUseSIWCForRequest(ctx, auth, steerHeaders) {
|
|
thinking.StripConfigurationUpdatesDoc(steerDoc)
|
|
}
|
|
steerCtx := helps.WithPayloadFinalizer(streamCtx, helps.NewPayloadFinalizer(e.cfg, "codex-websockets", thinking.ParseSuffix(req.Model).ModelName, "codex", "", steerReq.Payload, steerReq, steerOpts))
|
|
helps.FinalizePayloadDoc(steerCtx, steerDoc)
|
|
_ = steerDoc.Set("type", "response.steer")
|
|
parent := strings.Clone(steerDoc.Get("previous_response_id").String())
|
|
steerEventID := strings.Clone(steerDoc.Get("event_id").String())
|
|
payload = steerDoc.Bytes()
|
|
metadataMu.Lock()
|
|
settings := responseSettings[parent]
|
|
metadataMu.Unlock()
|
|
if settings == nil {
|
|
if !waitFor(func() bool { return len(pending) == 0 }) {
|
|
return
|
|
}
|
|
metadataMu.Lock()
|
|
settings = responseSettings[parent]
|
|
metadataMu.Unlock()
|
|
}
|
|
metadataMu.Lock()
|
|
unacknowledgedSteers = append(unacknowledgedSteers, codexSteerSubmission{parent: parent, eventID: steerEventID})
|
|
consumedPayload = nil
|
|
if settings != nil {
|
|
steeringSettings[parent] = settings
|
|
}
|
|
metadataMu.Unlock()
|
|
// Control frames bypass response.create translations and built-in defaults.
|
|
// Except for SIWC configuration_update stripping, unknown fields and unsupported
|
|
// input are left to upstream validation.
|
|
if !cliproxyexecutor.WebsocketAuthEnabled(streamCtx, auth.ID) {
|
|
fail(fmt.Errorf("websocket credential is no longer enabled"))
|
|
return
|
|
}
|
|
helps.RecordAPIWebsocketRequest(streamCtx, e.cfg, helps.UpstreamRequestLog{
|
|
URL: initial.wsURL, Method: "WEBSOCKET", Body: payload, Provider: e.Identifier(), AuthID: auth.ID,
|
|
})
|
|
if errWrite := writeCodexWebsocketMessage(sess, conn, payload); errWrite != nil {
|
|
fail(mapCodexWebsocketWriteError(sess, conn, errWrite))
|
|
return
|
|
}
|
|
log.Infof("codex websockets: request forwarded session=%s auth=%s url=%s event=response.steer", sess.sessionID, auth.ID, initial.wsURL)
|
|
continue
|
|
|
|
case "response.create", "response.append":
|
|
if len(pendingCreates) == 0 && readyForCreate() {
|
|
if !processCreatePayload(payload) {
|
|
return
|
|
}
|
|
} else {
|
|
if len(pendingCreates) >= 16 {
|
|
fail(fmt.Errorf("too many outstanding response.create requests"))
|
|
return
|
|
}
|
|
pendingCreates = append(pendingCreates, payload)
|
|
consumedPayload = nil
|
|
}
|
|
continue
|
|
|
|
default:
|
|
if !reject(fmt.Sprintf("unsupported websocket request type: %s", ojson.GetBytesNoCopy(payload, "type").String())) {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
go func() {
|
|
defer close(out)
|
|
defer func() {
|
|
cancel()
|
|
// Close releases a writer blocked in the network, then join it before
|
|
// releasing the execution session. No goroutine outlives its socket.
|
|
e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "duplex_closed", nil)
|
|
<-writerDone
|
|
sess.clearActive(conn, readCh)
|
|
unlock()
|
|
}()
|
|
send := func(chunk cliproxyexecutor.StreamChunk) bool {
|
|
select {
|
|
case out <- chunk:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
reporter := initialReporter
|
|
firstResponse := true
|
|
responseActive := false
|
|
outputItems := make(map[int64][]byte)
|
|
var outputFallback [][]byte
|
|
// A business rejection followed by EOF with nothing outstanding stays terminal.
|
|
terminalBusinessRejection := false
|
|
for {
|
|
kind, payload, errRead := readCodexWebsocketMessage(streamCtx, sess, conn, readCh)
|
|
if errRead != nil {
|
|
select {
|
|
case errWrite := <-writeErrors:
|
|
errRead = errWrite
|
|
default:
|
|
}
|
|
if ctx.Err() == nil {
|
|
var connectionErr error = &codexDuplexConnectionError{cause: errRead}
|
|
if lost := codexWebsocketTurnTransportError(errRead); cliproxyexecutor.IsUpstreamWebsocketTurnError(lost) {
|
|
// Stop and join the writer first: after that nothing else is sent
|
|
// upstream and the outcome of every operation is known.
|
|
cancel()
|
|
<-writerDone
|
|
metadataMu.Lock()
|
|
idle := !firstResponse && !responseActive && len(pending) == 0 && len(unacknowledgedSteers) == 0 && len(acceptedSteers) == 0 && len(pendingCreates) == 0 && len(consumedPayload) == 0
|
|
turnErr := &cliproxyexecutor.UpstreamWebsocketTurnError{Cause: errRead, Idle: idle}
|
|
if firstResponse || responseActive {
|
|
turnErr.EventID = strings.Clone(ojson.GetBytesNoCopy(current.originalPayload, "event_id").String())
|
|
}
|
|
abandon := func(payload []byte, notSent bool) {
|
|
if eventID := strings.Clone(ojson.GetBytesNoCopy(payload, "event_id").String()); eventID != "" {
|
|
turnErr.Abandoned = append(turnErr.Abandoned, &cliproxyexecutor.UpstreamWebsocketTurnError{EventID: eventID, NotSent: notSent})
|
|
}
|
|
}
|
|
for _, prepared := range pending {
|
|
if !(firstResponse && prepared == initial) { // the initial create owns the turn error
|
|
abandon(prepared.originalPayload, false)
|
|
}
|
|
}
|
|
for _, steer := range unacknowledgedSteers {
|
|
if steer.eventID != "" {
|
|
turnErr.Abandoned = append(turnErr.Abandoned, &cliproxyexecutor.UpstreamWebsocketTurnError{EventID: steer.eventID})
|
|
}
|
|
}
|
|
acceptedIDs := make([]string, 0, len(acceptedSteers))
|
|
for id := range acceptedSteers {
|
|
acceptedIDs = append(acceptedIDs, id)
|
|
}
|
|
sort.Strings(acceptedIDs)
|
|
for _, id := range acceptedIDs {
|
|
if eventID := acceptedSteers[id].eventID; eventID != "" {
|
|
turnErr.Abandoned = append(turnErr.Abandoned, &cliproxyexecutor.UpstreamWebsocketTurnError{EventID: eventID})
|
|
}
|
|
}
|
|
for _, queued := range pendingCreates {
|
|
abandon(queued, true)
|
|
}
|
|
abandon(consumedPayload, true)
|
|
if !firstResponse && !responseActive && len(turnErr.Abandoned) > 0 {
|
|
// No running turn owns the loss: the correlated errors report it.
|
|
turnErr.Idle = true
|
|
}
|
|
known := terminalBusinessRejection && !responseActive && len(pending) == 0 && len(unacknowledgedSteers) == 0 && len(acceptedSteers) == 0 && len(pendingCreates) == 0 && len(consumedPayload) == 0
|
|
metadataMu.Unlock()
|
|
if known {
|
|
// Already forwarded; EOF only ends the finished operation. Drop the loss marker.
|
|
connectionErr = &codexDuplexConnectionError{cause: errors.Unwrap(errRead)}
|
|
} else {
|
|
if sess != nil {
|
|
sess.markTerminalError(conn, turnErr)
|
|
e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "upstream_disconnected", turnErr)
|
|
}
|
|
connectionErr = turnErr
|
|
}
|
|
}
|
|
reporter.PublishFailure(ctx, connectionErr)
|
|
send(cliproxyexecutor.StreamChunk{Err: connectionErr})
|
|
}
|
|
return
|
|
}
|
|
if kind != websocket.TextMessage {
|
|
continue
|
|
}
|
|
payload = bytes.TrimSpace(payload)
|
|
eventType := ojson.GetBytesNoCopy(payload, "type").String()
|
|
establishing := firstResponse && eventType == "response.created"
|
|
if eventType == "response.created" {
|
|
metadataMu.Lock()
|
|
parent := strings.Clone(ojson.GetBytesNoCopy(payload, "response.previous_response_id").String())
|
|
if parent == "" {
|
|
parent = responseID
|
|
}
|
|
if !firstResponse && len(pending) == 0 {
|
|
settings := steeringSettings[parent]
|
|
if settings == nil {
|
|
settings = responseSettings[parent]
|
|
}
|
|
if settings == nil {
|
|
metadataMu.Unlock()
|
|
connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("automatic successor has no retained parent settings")}
|
|
reporter.PublishFailure(ctx, connectionErr)
|
|
send(cliproxyexecutor.StreamChunk{Err: connectionErr})
|
|
return
|
|
}
|
|
current = settings
|
|
}
|
|
createdResponseID := strings.Clone(ojson.GetBytesNoCopy(payload, "response.id").String())
|
|
for id, accepted := range acceptedSteers {
|
|
if accepted.parent == parent && accepted.successorID == "" {
|
|
accepted.successorID = createdResponseID
|
|
acceptedSteers[id] = accepted
|
|
}
|
|
}
|
|
waitingParent = ""
|
|
automaticActive = !firstResponse && len(pending) == 0
|
|
if len(pending) > 0 {
|
|
current, pending = pending[0], pending[1:]
|
|
}
|
|
responseID = createdResponseID
|
|
// Retain response settings, not request history or authorization headers.
|
|
// In-flight steering pins its parent's settings independently of this window.
|
|
snapshot := *current
|
|
snapshot.originalPayload, snapshot.wsHeaders = nil, nil
|
|
snapshotDoc := ojson.NewDoc([]byte("{}"))
|
|
if reasoning := ojson.GetBytesNoCopy(current.clientBody, "reasoning"); reasoning.Exists() {
|
|
_ = snapshotDoc.SetRawBytes("reasoning", []byte(reasoning.Raw))
|
|
}
|
|
if inst := ojson.GetBytesNoCopy(current.clientBody, "instructions"); inst.Exists() {
|
|
_ = snapshotDoc.SetRawBytes("instructions", []byte(inst.Raw))
|
|
}
|
|
if serviceTier := ojson.GetBytesNoCopy(current.clientBody, "service_tier"); serviceTier.Exists() {
|
|
_ = snapshotDoc.SetRawBytes("service_tier", []byte(serviceTier.Raw))
|
|
}
|
|
snapshot.clientBody = snapshotDoc.Bytes()
|
|
responseSettings[responseID] = &snapshot
|
|
responseOrder = append(responseOrder, responseID)
|
|
if len(responseOrder) > 16 {
|
|
delete(responseSettings, responseOrder[0])
|
|
responseOrder = responseOrder[1:]
|
|
}
|
|
releaseSteeringSettings(parent)
|
|
metadataMu.Unlock()
|
|
wakeWriter()
|
|
if !firstResponse {
|
|
reporter = helps.NewExecutorUsageReporter(followupCtx, e, req.Model, auth)
|
|
reporter.SetTranslatedReasoningEffort(current.clientBody, current.to.String())
|
|
reporter.StartResponseTTFT()
|
|
}
|
|
firstResponse = false
|
|
responseActive = true
|
|
outputItems = make(map[int64][]byte)
|
|
outputFallback = nil
|
|
}
|
|
observeCodexTokenEvent(reporter, payload)
|
|
helps.AppendCodexAPIWebsocketResponse(ctx, e.cfg, payload)
|
|
helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload)
|
|
// Steering acknowledgements, pending notifications and failures are opaque:
|
|
// preserve their IDs, input, sequence numbers and event types byte-for-byte.
|
|
if strings.HasPrefix(eventType, "response.steer.") {
|
|
id := strings.Clone(ojson.GetBytesNoCopy(payload, "steer.id").String())
|
|
parent := strings.Clone(ojson.GetBytesNoCopy(payload, "steer.previous_response_id").String())
|
|
if parent == "" {
|
|
parent = responseID
|
|
}
|
|
metadataMu.Lock()
|
|
consumeSubmission := func() codexSteerSubmission {
|
|
for i, target := range unacknowledgedSteers {
|
|
if target.parent == parent || target.parent == "" {
|
|
unacknowledgedSteers = append(unacknowledgedSteers[:i], unacknowledgedSteers[i+1:]...)
|
|
return target
|
|
}
|
|
}
|
|
return codexSteerSubmission{}
|
|
}
|
|
switch eventType {
|
|
case "response.steer.accepted":
|
|
acceptedSteers[id] = codexAcceptedSteer{parent: parent, eventID: consumeSubmission().eventID}
|
|
case "response.steer.failed":
|
|
if _, accepted := acceptedSteers[id]; accepted {
|
|
removeAcceptedSteer(id)
|
|
} else {
|
|
consumeSubmission()
|
|
}
|
|
releaseSteeringSettings(parent)
|
|
case "response.steer.pending":
|
|
// Tool results may already be waiting in the writer. No automatic
|
|
// successor can start until an explicit continuation supplies them.
|
|
waitingParent = parent
|
|
}
|
|
metadataMu.Unlock()
|
|
wakeWriter()
|
|
if !send(cliproxyexecutor.StreamChunk{Payload: payload}) {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
if !firstResponse && (eventType == "error" || eventType == "response.failed") {
|
|
var credentialErr error
|
|
if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok {
|
|
credentialErr = wsErr
|
|
} else if streamErr, _, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok {
|
|
credentialErr = streamErr
|
|
}
|
|
var status interface{ StatusCode() int }
|
|
if errors.As(credentialErr, &status) && (status.StatusCode() == http.StatusUnauthorized || status.StatusCode() == http.StatusForbidden || status.StatusCode() == http.StatusTooManyRequests) {
|
|
// Account health is independent of which queued request failed.
|
|
// The conductor records the original classification without replaying
|
|
// this already-started stream on another credential.
|
|
reporter.PublishFailure(ctx, credentialErr)
|
|
if send(cliproxyexecutor.StreamChunk{Payload: payload}) {
|
|
send(cliproxyexecutor.StreamChunk{Err: credentialErr})
|
|
}
|
|
return
|
|
}
|
|
}
|
|
eventPrepared, eventReporter := current, reporter
|
|
if !firstResponse && (eventType == "response.failed" || eventType == "error") {
|
|
failedID := ojson.GetBytesNoCopy(payload, "response.id").String()
|
|
if failedID == "" {
|
|
failedID = ojson.GetBytesNoCopy(payload, "response_id").String()
|
|
}
|
|
metadataMu.Lock()
|
|
// A failure for the running response must not consume a queued create.
|
|
// A rejection before response.created instead owns the oldest pending
|
|
// create, including its reasoning replay scope.
|
|
currentFailure := failedID != "" && failedID == responseID
|
|
ambiguous := failedID == "" && ((len(pending) > 0 && responseActive) || len(unacknowledgedSteers) > 0)
|
|
if len(pending) > 0 && !currentFailure && !ambiguous {
|
|
eventPrepared, pending = pending[0], pending[1:]
|
|
eventReporter = helps.NewExecutorUsageReporter(followupCtx, e, req.Model, auth)
|
|
eventReporter.SetTranslatedReasoningEffort(eventPrepared.clientBody, eventPrepared.to.String())
|
|
} else if !ambiguous {
|
|
responseActive = false
|
|
automaticActive = false
|
|
}
|
|
metadataMu.Unlock()
|
|
wakeWriter()
|
|
if ambiguous {
|
|
// Without a response ID, assigning this failure could corrupt
|
|
// either request. Preserve the event and fail the socket without
|
|
// guessing a scope, replaying input, or cooling the credential.
|
|
connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("cannot associate websocket failure with a response or pending create")}
|
|
reporter.PublishFailure(ctx, connectionErr)
|
|
if send(cliproxyexecutor.StreamChunk{Payload: payload}) {
|
|
send(cliproxyexecutor.StreamChunk{Err: connectionErr})
|
|
}
|
|
return
|
|
}
|
|
}
|
|
restoreMultiAgent := !eventPrepared.multiAgentV2Conflict && (eventPrepared.optimizeMultiAgentV2 || sess.isMultiAgentV2Optimized(conn))
|
|
payload = helps.RestoreCodexMultiAgentV2Response(payload, restoreMultiAgent)
|
|
// Parse and invalidate replay for every rejected request, using the
|
|
// metadata that belongs to this event. Only the first rejection can
|
|
// enter conductor bootstrap retry; later failures stay on this socket.
|
|
var terminalErr, replayErr error
|
|
if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok {
|
|
terminalErr = wsErr
|
|
replayErr = clearCodexReasoningReplayOnWebsocketError(ctx, eventPrepared.replayScope, payload)
|
|
} else if streamErr, body, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok {
|
|
terminalErr = streamErr
|
|
replayErr = clearCodexReasoningReplayOnInvalidSignature(ctx, eventPrepared.replayScope, streamErr.StatusCode(), body)
|
|
}
|
|
if replayErr != nil {
|
|
helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", replayErr)
|
|
eventReporter.PublishFailure(ctx, replayErr)
|
|
send(cliproxyexecutor.StreamChunk{Err: replayErr})
|
|
return
|
|
}
|
|
if terminalErr != nil {
|
|
var status interface{ StatusCode() int }
|
|
terminalBusinessRejection = errors.As(terminalErr, &status) && status.StatusCode() == http.StatusBadRequest
|
|
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", terminalErr)
|
|
eventReporter.PublishFailure(ctx, terminalErr)
|
|
if firstResponse {
|
|
send(cliproxyexecutor.StreamChunk{Err: terminalErr})
|
|
return
|
|
}
|
|
}
|
|
if eventType == "response.output_item.done" {
|
|
collectCodexOutputItemDone(payload, outputItems, &outputFallback)
|
|
}
|
|
if isCodexTerminalEventType(eventType) {
|
|
endedID := ojson.GetBytesNoCopy(payload, "response.id").String()
|
|
if endedID == "" {
|
|
endedID = ojson.GetBytesNoCopy(payload, "response_id").String()
|
|
}
|
|
if endedID == "" && !responseActive {
|
|
// An id-less failure that ended the running response.
|
|
endedID = responseID
|
|
}
|
|
metadataMu.Lock()
|
|
// A steer is complete once the response that consumed it has ended.
|
|
for id, accepted := range acceptedSteers {
|
|
if endedID != "" && accepted.successorID == endedID {
|
|
removeAcceptedSteer(id)
|
|
}
|
|
}
|
|
metadataMu.Unlock()
|
|
}
|
|
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
|
|
terminalBusinessRejection = false
|
|
responseActive = false
|
|
metadataMu.Lock()
|
|
automaticActive = false
|
|
metadataMu.Unlock()
|
|
wakeWriter()
|
|
payload = normalizeCodexWebsocketCompletion(payload)
|
|
if !current.preserveNativeOutput {
|
|
payload = patchCodexCompletedOutput(payload, outputItems, outputFallback)
|
|
}
|
|
if eventType != "response.incomplete" {
|
|
cacheCodexReasoningReplayFromCompleted(current.replayScope, payload)
|
|
}
|
|
if detail, ok := helps.ParseCodexUsageWithOutputItems(payload, outputItems, outputFallback); ok {
|
|
reporter.Publish(ctx, detail)
|
|
} else {
|
|
reporter.EnsurePublished(ctx)
|
|
}
|
|
publishCodexImageToolUsageWithOutputItems(ctx, reporter, current.clientBody, payload, outputItems, outputFallback)
|
|
}
|
|
if !send(cliproxyexecutor.StreamChunk{Payload: helps.EnsureResponsesUsageDetails(payload), Incomplete: eventType == "response.incomplete"}) {
|
|
return
|
|
}
|
|
if establishing {
|
|
// Deliver response.created before any locally generated error so the
|
|
// downstream handler also observes successful bootstrap first.
|
|
close(inputReady)
|
|
}
|
|
}
|
|
}()
|
|
return &cliproxyexecutor.StreamResult{Headers: headers, Chunks: out}
|
|
}
|
|
|
|
func isCodexTerminalEventType(eventType string) bool {
|
|
switch eventType {
|
|
case "response.completed", "response.done", "response.incomplete", "response.failed", "error":
|
|
return true
|
|
}
|
|
return false
|
|
}
|