Files
CLIProxyAPI/internal/runtime/executor/codex_websockets_session.go
Luis Pater 9717e325ba fix(responses): reject late websocket interrupts after terminal events
- Track the running response identity in the Codex websocket session and reject interrupts that do not target it, so an interrupt after response.completed is never forwarded upstream while turn teardown is pending.
- Gate the handler's local interrupt path by explicit turn ownership and close it before terminal payloads are written downstream, preventing fabricated response.incomplete events and silently dropped claimed frames.
- Add deterministic regression tests covering late and unknown interrupts across completion, failure, queued rejection, upstream loss, and HTTP fallback turns.

Closes: #7065
2026-10-28 18:33:51 +08:00

1222 lines
36 KiB
Go

package executor
import (
"bytes"
"context"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/gorilla/websocket"
"github.com/router-for-me/CLIProxyAPI/v8/internal/config"
cliproxyauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
log "github.com/sirupsen/logrus"
"github.com/tidwall/gjson"
)
type codexWebsocketSessionStore struct {
mu sync.Mutex
sessions map[string]*codexWebsocketSession
}
var globalCodexWebsocketSessionStore = &codexWebsocketSessionStore{
sessions: make(map[string]*codexWebsocketSession),
}
type websocketConnectionCloser struct {
conn *websocket.Conn
once sync.Once
err error
}
func newWebsocketConnectionCloser(conn *websocket.Conn) *websocketConnectionCloser {
if conn == nil {
return nil
}
return &websocketConnectionCloser{conn: conn}
}
func (c *websocketConnectionCloser) Close() error {
if c == nil || c.conn == nil {
return nil
}
c.once.Do(func() {
c.err = c.conn.Close()
})
return c.err
}
type codexWebsocketSession struct {
sessionID string
reqMu sync.Mutex
connMu sync.Mutex
// recoverTransport (under connMu): the current turn belongs to a downstream websocket,
// so a transport loss ends the turn but not the downstream socket.
recoverTransport bool
turnContext context.Context // attempt context of the current downstream turn
conn *websocket.Conn
connCloser *websocketConnectionCloser
wsURL string
authID string
proxyURL string
useSIWC bool
connCfg *config.Config
hasConnCfg bool
multiAgentV2OptimizedConn *websocket.Conn
lifecycleBindMu sync.Mutex
lifecycle cliproxyexecutor.ExecutionLifecycle
lifecycleModel string
writeMu sync.Mutex
activeMu sync.Mutex
activeConn *websocket.Conn
activeCh chan codexWebsocketRead
activeDone <-chan struct{}
activeCancel context.CancelFunc
terminalConn *websocket.Conn
terminalErr error
responseStateConn *websocket.Conn
terminalResponseIDs map[string]struct{}
// activeResponseID is the newest response known to run, or to have run,
// on responseStateConn. The read loop sets it on response.created and on
// the terminal event of that same response, before the payload is
// delivered downstream. activeResponseTerminal then tells an interrupt
// snapshot that the turn already ended even while its read channel is
// still awaiting teardown. A duplex socket may also reject a queued
// create while another response runs; only the matching id ends the
// active response. An empty id means no response.created was seen yet.
activeResponseID string
activeResponseTerminal bool
readerConn *websocket.Conn
upstreamDisconnectOnce sync.Once
upstreamDisconnectCh chan error
upstreamDisconnectErrMu sync.RWMutex
upstreamDisconnectErrConn *websocket.Conn
upstreamDisconnectErr error
lastEventMu sync.Mutex
lastEventConn *websocket.Conn
lastEventType string
}
type codexWebsocketRead struct {
conn *websocket.Conn
msgType int
payload []byte
err error
}
func (s *codexWebsocketSession) setActive(conn *websocket.Conn, ch chan codexWebsocketRead) {
if s == nil {
return
}
s.activeMu.Lock()
s.setActiveLocked(conn, ch)
s.activeMu.Unlock()
}
func (s *codexWebsocketSession) setActiveLocked(conn *websocket.Conn, ch chan codexWebsocketRead) {
if s.activeCancel != nil {
s.activeCancel()
s.activeCancel = nil
s.activeDone = nil
}
s.activeConn = conn
s.activeCh = ch
s.activeResponseID = ""
s.activeResponseTerminal = false
if conn != nil && ch != nil {
activeCtx, activeCancel := context.WithCancel(context.Background())
s.activeDone = activeCtx.Done()
s.activeCancel = activeCancel
}
}
func (s *codexWebsocketSession) activate(conn *websocket.Conn) chan codexWebsocketRead {
if s == nil || conn == nil {
return nil
}
ch := make(chan codexWebsocketRead, 4096)
s.activeMu.Lock()
defer s.activeMu.Unlock()
if s.terminalConn == conn && s.terminalErr != nil {
ch <- codexWebsocketRead{conn: conn, err: s.terminalErr}
close(ch)
return ch
}
s.setActiveLocked(conn, ch)
return ch
}
func (s *codexWebsocketSession) resetTerminalError(conn *websocket.Conn) {
if s == nil || conn == nil {
return
}
s.activeMu.Lock()
s.terminalConn = conn
s.terminalErr = nil
if s.responseStateConn != conn {
s.responseStateConn = conn
s.terminalResponseIDs = make(map[string]struct{})
}
s.activeMu.Unlock()
}
func (s *codexWebsocketSession) markTerminalError(conn *websocket.Conn, err error) bool {
if s == nil || conn == nil || err == nil {
return false
}
s.connMu.Lock()
if s.conn != conn {
s.connMu.Unlock()
return false
}
s.activeMu.Lock()
s.terminalConn = conn
s.terminalErr = err
s.activeMu.Unlock()
s.connMu.Unlock()
return true
}
func (s *codexWebsocketSession) activeForConn(conn *websocket.Conn) (chan codexWebsocketRead, <-chan struct{}) {
if s == nil || conn == nil {
return nil, nil
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
if s.activeConn != conn {
return nil, nil
}
return s.activeCh, s.activeDone
}
func clearRetryActiveState(sess *codexWebsocketSession, conn *websocket.Conn, ch chan codexWebsocketRead) bool {
if sess == nil {
return false
}
return sess.clearActive(conn, ch)
}
func (s *codexWebsocketSession) clearActive(conn *websocket.Conn, ch chan codexWebsocketRead) bool {
if s == nil {
return false
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
if s.activeConn != conn || s.activeCh != ch {
return false
}
s.activeConn = nil
s.activeCh = nil
s.activeResponseID = ""
s.activeResponseTerminal = false
if s.activeCancel != nil {
s.activeCancel()
}
s.activeCancel = nil
s.activeDone = nil
return true
}
const codexWebsocketWriteChunkSize = 32 * 1024
var (
testWebsocketWritePayloadHook func(conn *websocket.Conn)
testWebsocketWriteChunkHook func(chunkIndex int, totalChunks int)
)
func (s *codexWebsocketSession) writeMessage(conn *websocket.Conn, msgType int, payload []byte) error {
if s == nil {
return fmt.Errorf("codex websockets executor: session is nil")
}
if conn == nil {
return fmt.Errorf("codex websockets executor: websocket conn is nil")
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
if testWebsocketWritePayloadHook != nil {
testWebsocketWritePayloadHook(conn)
}
if len(payload) <= codexWebsocketWriteChunkSize {
return conn.WriteMessage(msgType, payload)
}
w, errNext := conn.NextWriter(msgType)
if errNext != nil {
return errNext
}
totalChunks := (len(payload) + codexWebsocketWriteChunkSize - 1) / codexWebsocketWriteChunkSize
for i := 0; i < len(payload); i += codexWebsocketWriteChunkSize {
end := i + codexWebsocketWriteChunkSize
if end > len(payload) {
end = len(payload)
}
chunkIdx := i / codexWebsocketWriteChunkSize
if testWebsocketWriteChunkHook != nil {
testWebsocketWriteChunkHook(chunkIdx, totalChunks)
}
if _, errWrite := w.Write(payload[i:end]); errWrite != nil {
_ = w.Close()
return errWrite
}
}
return w.Close()
}
func (s *codexWebsocketSession) setMultiAgentV2Optimized(conn *websocket.Conn, optimized bool) {
if s == nil || conn == nil {
return
}
s.connMu.Lock()
if s.conn == conn {
if optimized {
s.multiAgentV2OptimizedConn = conn
} else {
s.multiAgentV2OptimizedConn = nil
}
}
s.connMu.Unlock()
}
func (s *codexWebsocketSession) isMultiAgentV2Optimized(conn *websocket.Conn) bool {
if s == nil || conn == nil {
return false
}
s.connMu.Lock()
defer s.connMu.Unlock()
return s.conn == conn && s.multiAgentV2OptimizedConn == conn
}
// sendTerminalWebsocketRead reports whether it invalidated a full channel's connection before waiting.
func sendTerminalWebsocketRead(ch chan<- codexWebsocketRead, done <-chan struct{}, event codexWebsocketRead, invalidate func()) bool {
select {
case ch <- event:
return false
case <-done:
return false
default:
}
invalidated := invalidate != nil
if invalidated {
invalidate()
}
select {
case ch <- event:
case <-done:
}
return invalidated
}
func (s *codexWebsocketSession) configureConn(conn *websocket.Conn) {
if s == nil || conn == nil {
return
}
s.resetTerminalError(conn)
s.resetUpstreamDisconnectError(conn)
conn.SetPingHandler(func(appData string) error {
sessionID := ""
if s != nil {
sessionID = s.sessionID
}
sessionKind := sessionObjectKind(s)
log.Debugf("codex websockets: upstream ping received session=%s session_object=%s ping_bytes=%d", sessionID, sessionKind, len(appData))
log.Debugf("codex websockets: upstream pong write started session=%s session_object=%s", sessionID, sessionKind)
start := time.Now()
// Gorilla websocket allows concurrent WriteControl with WriteMessage.
// Avoid writeMu here so keepalive pongs are not starved by long payload writes.
errPong := conn.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(10*time.Second))
if errPong != nil {
log.Warnf("codex websockets: upstream pong write failed session=%s session_object=%s duration=%v err=%v", sessionID, sessionKind, time.Since(start), errPong)
} else {
log.Debugf("codex websockets: upstream pong replied session=%s session_object=%s duration=%v", sessionID, sessionKind, time.Since(start))
}
return errPong
})
defaultCloseHandler := conn.CloseHandler()
conn.SetCloseHandler(func(code int, text string) error {
s.setUpstreamDisconnectError(conn, &websocket.CloseError{Code: code, Text: text})
return defaultCloseHandler(code, text)
})
}
func (s *codexWebsocketSession) bindExecutionLifecycle(opts cliproxyexecutor.Options, conn *websocket.Conn, closer *websocketConnectionCloser, model string) error {
if closer == nil {
return fmt.Errorf("codex websockets executor: websocket connection closer is nil")
}
if s == nil {
return cliproxyexecutor.BindExecutionResource(opts, closer)
}
lifecycle := opts.ExecutionLifecycle
if lifecycle == nil || conn == nil {
return nil
}
s.lifecycleBindMu.Lock()
defer s.lifecycleBindMu.Unlock()
s.connMu.Lock()
if s.conn == conn && s.connCloser == nil {
s.connCloser = closer
}
alreadyBound := s.conn == conn && s.connCloser == closer && s.lifecycle == lifecycle
s.connMu.Unlock()
if alreadyBound {
return nil
}
if errBind := lifecycle.Bind(func() error {
return s.closeBoundConnection(conn, closer, lifecycle)
}); errBind != nil {
return errBind
}
if retained, ok := lifecycle.(interface{ Retain() }); ok {
retained.Retain()
}
s.connMu.Lock()
if s.conn != conn || s.connCloser != closer {
s.connMu.Unlock()
return fmt.Errorf("codex websockets executor: websocket connection closed during lifecycle bind")
}
previous := s.lifecycle
s.lifecycle = lifecycle
s.lifecycleModel = strings.TrimSpace(model)
s.connMu.Unlock()
if previous != nil && previous != lifecycle {
previous.End("target_replaced")
}
return nil
}
func (s *codexWebsocketSession) closeBoundConnection(conn *websocket.Conn, closer *websocketConnectionCloser, lifecycle cliproxyexecutor.ExecutionLifecycle) error {
if s == nil || conn == nil {
return nil
}
s.detachConnection(conn, lifecycle)
errClose := closer.Close()
go lifecycle.End("connection_closed")
return errClose
}
func (s *codexWebsocketSession) detachConnection(conn *websocket.Conn, lifecycle cliproxyexecutor.ExecutionLifecycle) *websocketConnectionCloser {
if s == nil || conn == nil {
return nil
}
s.connMu.Lock()
var closer *websocketConnectionCloser
matched := s.conn == conn
if matched {
closer = s.connCloser
s.conn = nil
s.connCloser = nil
s.connCfg = nil
s.hasConnCfg = false
s.multiAgentV2OptimizedConn = nil
if s.readerConn == conn {
s.readerConn = nil
}
}
if (lifecycle == nil && matched) || (lifecycle != nil && s.lifecycle == lifecycle) {
s.lifecycle = nil
s.lifecycleModel = ""
}
s.connMu.Unlock()
return closer
}
func closeWebsocketAfterBindFailure(sess *codexWebsocketSession, conn *websocket.Conn, closer *websocketConnectionCloser) {
if conn == nil || closer == nil {
return
}
if sess != nil {
sess.detachConnection(conn, nil)
}
if errClose := closer.Close(); errClose != nil {
log.Errorf("websockets executor: close lifecycle bind failure connection error: %v", errClose)
}
}
func websocketSessionTargetChanged(sess *codexWebsocketSession, authID string, wsURL string, proxyURL string, useSIWC ...bool) bool {
if sess == nil {
return false
}
sess.connMu.Lock()
defer sess.connMu.Unlock()
if strings.TrimSpace(sess.authID) == "" && strings.TrimSpace(sess.wsURL) == "" {
return false
}
return !websocketSessionTargetMatches(sess, authID, wsURL, proxyURL, useSIWC...)
}
func existingWebsocketSessionConn(sess *codexWebsocketSession, authID string, wsURL string, proxyURL string, useSIWC ...bool) (*websocket.Conn, *websocketConnectionCloser) {
if sess == nil {
return nil, nil
}
sess.connMu.Lock()
conn := sess.conn
closer := sess.connCloser
matches := conn != nil && closer != nil && websocketSessionTargetMatches(sess, authID, wsURL, proxyURL, useSIWC...)
sess.connMu.Unlock()
if !matches || sess.upstreamDisconnectError(conn) != nil {
return nil, nil
}
return conn, closer
}
func (s *codexWebsocketSession) effectiveConfig(authID string, fallback *config.Config) *config.Config {
if s == nil {
return fallback
}
s.connMu.Lock()
sameConn := s.conn != nil && s.hasConnCfg && strings.TrimSpace(s.authID) == strings.TrimSpace(authID)
cfg := s.connCfg
s.connMu.Unlock()
if sameConn {
return cfg
}
return fallback
}
func websocketSessionTargetMatches(sess *codexWebsocketSession, authID string, wsURL string, proxyURL string, useSIWC ...bool) bool {
if len(useSIWC) > 0 && sess.useSIWC != useSIWC[0] {
return false
}
return strings.TrimSpace(sess.authID) == strings.TrimSpace(authID) &&
strings.TrimSpace(sess.wsURL) == strings.TrimSpace(wsURL) &&
strings.TrimSpace(sess.proxyURL) == strings.TrimSpace(proxyURL)
}
func codexWebsocketRequestUsesSIWC(ctx context.Context, auth *cliproxyauth.Auth, headers http.Header, explicitUseSIWC ...bool) bool {
if len(explicitUseSIWC) > 0 {
return explicitUseSIWC[0]
}
if !codexUseSIWC(auth) {
return false
}
if headers != nil {
if authHeader := strings.TrimSpace(headers.Get("Authorization")); strings.HasPrefix(strings.ToLower(authHeader), "bearer ") {
tok := strings.TrimSpace(authHeader[len("Bearer "):])
if siwcTok := codexSIWCAccessToken(auth); siwcTok != "" && tok != "" {
return tok == siwcTok
}
}
}
return codexUseSIWCForRequest(ctx, auth, headers)
}
func detachMismatchedWebsocketSessionConn(sess *codexWebsocketSession, authID string, wsURL string, proxyURL string, useSIWC ...bool) (*websocket.Conn, *websocketConnectionCloser, string, string, cliproxyexecutor.ExecutionLifecycle) {
if sess == nil {
return nil, nil, "", "", nil
}
sess.connMu.Lock()
defer sess.connMu.Unlock()
conn := sess.conn
if conn == nil || websocketSessionTargetMatches(sess, authID, wsURL, proxyURL, useSIWC...) {
return nil, nil, "", "", nil
}
previousAuthID := sess.authID
previousWSURL := sess.wsURL
lifecycle := sess.lifecycle
closer := sess.connCloser
sess.lifecycle = nil
sess.lifecycleModel = ""
sess.conn = nil
sess.connCloser = nil
sess.connCfg = nil
sess.hasConnCfg = false
sess.useSIWC = false
sess.multiAgentV2OptimizedConn = nil
if sess.readerConn == conn {
sess.readerConn = nil
}
return conn, closer, previousAuthID, previousWSURL, lifecycle
}
func configureRawCodexWebsocketConn(conn *websocket.Conn, authID string, wsURL string) {
if conn == nil {
return
}
conn.SetPingHandler(func(appData string) error {
log.Debugf("codex websockets: upstream ping received session= session_object=none ping_bytes=%d", len(appData))
log.Debugf("codex websockets: upstream pong write started session= session_object=none")
start := time.Now()
errPong := conn.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(10*time.Second))
if errPong != nil {
log.Warnf("codex websockets: upstream pong write failed session= session_object=none duration=%v err=%v", time.Since(start), errPong)
} else {
log.Debugf("codex websockets: upstream pong replied session= session_object=none duration=%v", time.Since(start))
}
return errPong
})
}
func (s *codexWebsocketSession) setLastEventType(conn *websocket.Conn, eventType string) {
if s == nil || conn == nil || eventType == "" {
return
}
s.lastEventMu.Lock()
if s.lastEventConn == conn {
s.lastEventType = eventType
}
s.lastEventMu.Unlock()
}
func (s *codexWebsocketSession) getLastEventType(conn *websocket.Conn) string {
if s == nil || conn == nil {
return ""
}
s.lastEventMu.Lock()
defer s.lastEventMu.Unlock()
if s.lastEventConn != conn {
return ""
}
return s.lastEventType
}
func newEphemeralCodexWebsocketSession() *codexWebsocketSession {
return &codexWebsocketSession{
sessionID: "",
upstreamDisconnectCh: make(chan error, 1),
}
}
func (s *codexWebsocketSession) resetUpstreamDisconnectError(conn *websocket.Conn) {
if s == nil || conn == nil {
return
}
s.upstreamDisconnectErrMu.Lock()
s.upstreamDisconnectErrConn = conn
s.upstreamDisconnectErr = nil
s.upstreamDisconnectErrMu.Unlock()
s.lastEventMu.Lock()
s.lastEventConn = conn
s.lastEventType = ""
s.lastEventMu.Unlock()
}
func (s *codexWebsocketSession) setUpstreamDisconnectError(conn *websocket.Conn, err error) {
if s == nil || conn == nil || err == nil {
return
}
s.upstreamDisconnectErrMu.Lock()
if s.upstreamDisconnectErrConn == conn && s.upstreamDisconnectErr == nil {
s.upstreamDisconnectErr = err
}
s.upstreamDisconnectErrMu.Unlock()
}
func (s *codexWebsocketSession) upstreamDisconnectError(conn *websocket.Conn) error {
if s == nil || conn == nil {
return nil
}
s.upstreamDisconnectErrMu.RLock()
defer s.upstreamDisconnectErrMu.RUnlock()
if s.upstreamDisconnectErrConn != conn {
return nil
}
return s.upstreamDisconnectErr
}
func (s *codexWebsocketSession) notifyUpstreamDisconnect(err error) {
if s == nil {
return
}
if cliproxyexecutor.IsUpstreamWebsocketTurnError(err) {
// The active turn reports it; an idle loss ends no turn and keeps the downstream socket.
return
}
s.upstreamDisconnectOnce.Do(func() {
if s.upstreamDisconnectCh == nil {
return
}
select {
case s.upstreamDisconnectCh <- err:
default:
}
close(s.upstreamDisconnectCh)
})
}
func executionSessionIDFromOptions(opts cliproxyexecutor.Options) string {
if len(opts.Metadata) == 0 {
return ""
}
raw, ok := opts.Metadata[cliproxyexecutor.ExecutionSessionMetadataKey]
if !ok || raw == nil {
return ""
}
switch v := raw.(type) {
case string:
return strings.TrimSpace(v)
case []byte:
return strings.TrimSpace(string(v))
default:
return ""
}
}
func (e *CodexWebsocketsExecutor) getOrCreateSession(sessionID string) *codexWebsocketSession {
sessionID = strings.TrimSpace(sessionID)
if sessionID == "" {
return nil
}
if e == nil {
return nil
}
store := e.store
if store == nil {
store = globalCodexWebsocketSessionStore
}
store.mu.Lock()
defer store.mu.Unlock()
if store.sessions == nil {
store.sessions = make(map[string]*codexWebsocketSession)
}
if sess, ok := store.sessions[sessionID]; ok && sess != nil {
return sess
}
sess := &codexWebsocketSession{
sessionID: sessionID,
upstreamDisconnectCh: make(chan error, 1),
}
store.sessions[sessionID] = sess
return sess
}
func (e *CodexWebsocketsExecutor) UpstreamDisconnectChan(sessionID string) <-chan error {
sess := e.getOrCreateSession(sessionID)
if sess == nil {
return nil
}
return sess.upstreamDisconnectCh
}
func (e *CodexWebsocketsExecutor) ensureUpstreamConn(ctx context.Context, auth *cliproxyauth.Auth, sess *codexWebsocketSession, authID string, wsURL string, headers http.Header) (*websocket.Conn, *websocketConnectionCloser, *http.Response, error) {
return e.ensureUpstreamConnWithSIWC(ctx, auth, sess, authID, wsURL, headers, codexWebsocketRequestUsesSIWC(ctx, auth, headers))
}
func (e *CodexWebsocketsExecutor) ensureUpstreamConnWithSIWC(ctx context.Context, auth *cliproxyauth.Auth, sess *codexWebsocketSession, authID string, wsURL string, headers http.Header, useSIWC bool) (*websocket.Conn, *websocketConnectionCloser, *http.Response, error) {
if sess == nil {
conn, closer, resp, err := e.dialCodexWebsocket(ctx, auth, wsURL, headers)
if conn != nil {
configureRawCodexWebsocketConn(conn, authID, wsURL)
}
return conn, closer, resp, err
}
proxyURL := executionProxyURL(ctx, sess.effectiveConfig(authID, e.cfg), auth)
if staleConn, staleCloser, staleAuthID, staleWSURL, staleLifecycle := detachMismatchedWebsocketSessionConn(sess, authID, wsURL, proxyURL, useSIWC); staleConn != nil {
staleLastEvent := sess.getLastEventType(staleConn)
logCodexWebsocketDisconnectedWithLastEvent(sess, sess.sessionID, staleAuthID, staleWSURL, "target_changed", staleLastEvent, nil)
if staleCloser != nil {
if errClose := staleCloser.Close(); errClose != nil {
log.Errorf("codex websockets executor: close stale websocket error: %v", errClose)
}
}
if staleLifecycle != nil {
staleLifecycle.End("target_changed")
}
}
sess.connMu.Lock()
conn := sess.conn
closer := sess.connCloser
readerConn := sess.readerConn
sess.connMu.Unlock()
if conn != nil {
if readerConn != conn {
sess.connMu.Lock()
sess.readerConn = conn
sess.connMu.Unlock()
sess.configureConn(conn)
go e.readUpstreamLoop(sess, conn)
}
logCodexWebsocketConnectedWithReused(sess, sess.sessionID, authID, wsURL, true)
return conn, closer, nil, nil
}
if errLoss := cliproxyexecutor.WebsocketInputLossError(ctx); errLoss != nil {
return nil, nil, nil, errLoss
}
dialProxyURL := executionProxyURL(ctx, e.cfg, auth)
conn, closer, resp, errDial := e.dialCodexWebsocket(ctx, auth, wsURL, headers)
if errDial != nil {
return nil, closer, resp, errDial
}
sess.connMu.Lock()
if sess.conn != nil && websocketSessionTargetMatches(sess, authID, wsURL, dialProxyURL, useSIWC) {
previous := sess.conn
previousCloser := sess.connCloser
sess.connMu.Unlock()
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
logCodexWebsocketConnectedWithReused(sess, sess.sessionID, authID, wsURL, true)
return previous, previousCloser, nil, nil
}
sess.conn = conn
sess.connCloser = closer
sess.connCfg = e.cfg
sess.hasConnCfg = true
sess.multiAgentV2OptimizedConn = nil
sess.wsURL = wsURL
sess.authID = authID
sess.proxyURL = dialProxyURL
sess.useSIWC = useSIWC
sess.readerConn = conn
sess.connMu.Unlock()
sess.configureConn(conn)
go e.readUpstreamLoop(sess, conn)
logCodexWebsocketConnectedWithReused(sess, sess.sessionID, authID, wsURL, false)
return conn, closer, resp, nil
}
func (e *CodexWebsocketsExecutor) readUpstreamLoop(sess *codexWebsocketSession, conn *websocket.Conn) {
if e == nil || sess == nil || conn == nil {
return
}
for {
_ = conn.SetReadDeadline(time.Now().Add(codexResponsesWebsocketIdleTimeout))
msgType, payload, errRead := conn.ReadMessage()
if errRead != nil {
sess.connMu.Lock()
recoverTransport := sess.recoverTransport
sess.connMu.Unlock()
if recoverTransport {
errRead = codexWebsocketTurnTransportError(errRead)
}
sess.markTerminalError(conn, errRead)
invalidate := func() {
e.invalidateUpstreamConn(sess, conn, "upstream_disconnected", errRead)
}
invalidated := false
if cliproxyexecutor.IsUpstreamWebsocketTurnError(errRead) {
// Drop the dead connection before the turn error is visible, so the next
// create reconnects instead of writing to it.
invalidate()
invalidated = true
}
ch, done := sess.activeForConn(conn)
if ch != nil {
invalidated = sendTerminalWebsocketRead(ch, done, codexWebsocketRead{conn: conn, err: errRead}, invalidate)
if sess.clearActive(conn, ch) {
close(ch)
}
}
if !invalidated {
invalidate()
}
return
}
if msgType != websocket.TextMessage {
if msgType == websocket.BinaryMessage {
errBinary := fmt.Errorf("codex websockets executor: unexpected binary message")
sess.markTerminalError(conn, errBinary)
invalidate := func() {
e.invalidateUpstreamConn(sess, conn, "unexpected_binary", errBinary)
}
invalidated := false
ch, done := sess.activeForConn(conn)
if ch != nil {
invalidated = sendTerminalWebsocketRead(ch, done, codexWebsocketRead{conn: conn, err: errBinary}, invalidate)
if sess.clearActive(conn, ch) {
close(ch)
}
}
if !invalidated {
invalidate()
}
return
}
continue
}
payload = bytes.TrimSpace(payload)
if len(payload) > 0 {
eventType := gjson.GetBytes(payload, "type").String()
if eventType != "" {
sess.setLastEventType(conn, eventType)
}
// Track the running response before delivery: downstream may
// still be processing this turn after its active read channel is
// cleared, and interrupts arriving after the completion must not
// forward while teardown is still pending. Terminal events only
// end their own response; a duplex socket can reject a queued
// create while another response is still running.
//
// Failure events (error, response.failed) use the duplex
// attribution shapes: an explicit response id (response.id or
// top-level response_id) ends only that response, while an
// id-less failure ends the running one. Success terminals only
// carry response.id, and one without an id is left unattributed,
// as before.
responseID := gjson.GetBytes(payload, "response.id").String()
if eventType == "response.created" {
if responseID != "" {
sess.activeMu.Lock()
if sess.responseStateConn == conn && sess.activeConn == conn && sess.activeCh != nil {
sess.activeResponseID = responseID
sess.activeResponseTerminal = false
}
sess.activeMu.Unlock()
}
} else if isTerminalEvent(eventType) {
failure := eventType == "error" || eventType == "response.failed"
if failure && responseID == "" {
responseID = gjson.GetBytes(payload, "response_id").String()
}
if responseID != "" || failure {
sess.activeMu.Lock()
if sess.responseStateConn == conn {
terminalID := responseID
if terminalID == "" {
// Id-less failure: the running response ended.
terminalID = sess.activeResponseID
}
if terminalID != "" && eventType != "error" {
// Match the pre-existing terminal-ID recording,
// which never covered error events.
sess.terminalResponseIDs[terminalID] = struct{}{}
}
if terminalID != "" && sess.activeResponseID == terminalID {
sess.activeResponseTerminal = true
}
}
sess.activeMu.Unlock()
}
}
}
ch, done := sess.activeForConn(conn)
if ch == nil {
continue
}
select {
case ch <- codexWebsocketRead{conn: conn, msgType: msgType, payload: payload}:
case <-done:
}
}
}
func (e *CodexWebsocketsExecutor) invalidateUpstreamConn(sess *codexWebsocketSession, conn *websocket.Conn, reason string, err error) {
e.invalidateUpstreamConnWithNotify(sess, conn, reason, err, true)
}
func (e *CodexWebsocketsExecutor) invalidateUpstreamConnWithoutDisconnectNotify(sess *codexWebsocketSession, conn *websocket.Conn, reason string, err error) {
e.invalidateUpstreamConnWithNotify(sess, conn, reason, err, false)
}
func (e *CodexWebsocketsExecutor) invalidateUpstreamConnWithNotify(sess *codexWebsocketSession, conn *websocket.Conn, reason string, err error, notify bool) {
if sess == nil || conn == nil {
return
}
sess.connMu.Lock()
current := sess.conn
authID := sess.authID
wsURL := sess.wsURL
sessionID := sess.sessionID
if current == nil || current != conn {
sess.connMu.Unlock()
return
}
lifecycle := sess.lifecycle
turnContext := sess.turnContext
sess.turnContext = nil
closer := sess.connCloser
sess.lifecycle = nil
sess.lifecycleModel = ""
sess.conn = nil
sess.connCloser = nil
sess.connCfg = nil
sess.hasConnCfg = false
sess.multiAgentV2OptimizedConn = nil
if sess.readerConn == conn {
sess.readerConn = nil
}
if cliproxyexecutor.IsUpstreamWebsocketTurnError(err) {
cliproxyexecutor.NotifyWebsocketInputLoss(turnContext)
}
sess.connMu.Unlock()
lastEvent := sess.getLastEventType(conn)
logCodexWebsocketDisconnectedWithLastEvent(sess, sessionID, authID, wsURL, reason, lastEvent, err)
if notify {
sess.notifyUpstreamDisconnect(err)
}
if closer != nil {
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
}
if lifecycle != nil {
if owner, ok := lifecycle.(interface{ EndWebsocketTurn(context.Context, string) }); ok && cliproxyexecutor.IsUpstreamWebsocketTurnError(err) {
// The lost turn's attempt must survive to deliver its error to the client.
owner.EndWebsocketTurn(turnContext, reason)
} else {
lifecycle.End(reason)
}
}
}
func (e *CodexWebsocketsExecutor) sessionStore() *codexWebsocketSessionStore {
if e == nil {
return nil
}
if e.store != nil {
return e.store
}
return globalCodexWebsocketSessionStore
}
func (e *CodexWebsocketsExecutor) SharesExecutionSessionsWith(other cliproxyauth.ProviderExecutor) bool {
if e == nil || other == nil {
return false
}
store := e.sessionStore()
if store == nil {
return false
}
switch next := other.(type) {
case *CodexWebsocketsExecutor:
return next != nil && store == next.sessionStore()
case *CodexAutoExecutor:
return next != nil && next.wsExec != nil && store == next.wsExec.sessionStore()
default:
return false
}
}
func (e *CodexWebsocketsExecutor) CloseAuthExecutionSessions(authID string) {
if e == nil {
return
}
closeCodexWebsocketSessionsInStoreForAuthID(e.sessionStore(), authID, "auth_removed")
}
func (e *CodexWebsocketsExecutor) CloseExecutionSession(sessionID string) {
sessionID = strings.TrimSpace(sessionID)
if e == nil {
return
}
if sessionID == "" {
return
}
if sessionID == cliproxyauth.CloseAllExecutionSessionsID {
e.closeAllExecutionSessions("executor_shutdown")
return
}
store := e.sessionStore()
store.mu.Lock()
sess := store.sessions[sessionID]
delete(store.sessions, sessionID)
store.mu.Unlock()
e.closeExecutionSession(sess, "session_closed")
}
func (e *CodexWebsocketsExecutor) closeAllExecutionSessions(reason string) {
if e == nil {
return
}
store := e.store
if store == nil {
store = globalCodexWebsocketSessionStore
}
store.mu.Lock()
sessions := make([]*codexWebsocketSession, 0, len(store.sessions))
for sessionID, sess := range store.sessions {
delete(store.sessions, sessionID)
if sess != nil {
sessions = append(sessions, sess)
}
}
store.mu.Unlock()
for i := range sessions {
e.closeExecutionSession(sessions[i], reason)
}
}
func (e *CodexWebsocketsExecutor) closeExecutionSession(sess *codexWebsocketSession, reason string) {
closeCodexWebsocketSession(sess, reason)
}
func closeCodexWebsocketSession(sess *codexWebsocketSession, reason string) {
if sess == nil {
return
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "session_closed"
}
sess.connMu.Lock()
conn := sess.conn
authID := sess.authID
wsURL := sess.wsURL
lifecycle := sess.lifecycle
closer := sess.connCloser
sess.lifecycle = nil
sess.lifecycleModel = ""
sess.conn = nil
sess.connCloser = nil
sess.connCfg = nil
sess.hasConnCfg = false
sess.multiAgentV2OptimizedConn = nil
if sess.readerConn == conn {
sess.readerConn = nil
}
sessionID := sess.sessionID
sess.connMu.Unlock()
lastEvent := sess.getLastEventType(conn)
if conn != nil {
logCodexWebsocketDisconnectedWithLastEvent(sess, sessionID, authID, wsURL, reason, lastEvent, nil)
if closer != nil {
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
}
}
if lifecycle != nil {
lifecycle.End(reason)
}
}
func sessionObjectKind(sess *codexWebsocketSession) string {
if sess == nil {
return "none"
}
if strings.TrimSpace(sess.sessionID) != "" {
return "persistent"
}
return "ephemeral"
}
func logCodexWebsocketConnected(sessionID string, authID string, wsURL string) {
logCodexWebsocketConnectedWithReused(nil, sessionID, authID, wsURL, false)
}
func logCodexWebsocketConnectedWithReused(sess *codexWebsocketSession, sessionID string, authID string, wsURL string, reused bool) {
sessionStr := strings.TrimSpace(sessionID)
sessionKind := sessionObjectKind(sess)
if reused {
log.Infof("codex websockets: upstream connected session=%s auth=%s url=%s session_object=%s reused=true", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind)
return
}
log.Infof("codex websockets: upstream connected session=%s auth=%s url=%s session_object=%s reused=false", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind)
}
func logCodexWebsocketDisconnected(sessionID string, authID string, wsURL string, reason string, err error) {
logCodexWebsocketDisconnectedWithLastEvent(nil, sessionID, authID, wsURL, reason, "", err)
}
func isTerminalEvent(eventType string) bool {
switch eventType {
case "response.completed", "response.done", "response.incomplete", "response.failed", "error":
return true
default:
return false
}
}
func logCodexWebsocketDisconnectedWithLastEvent(sess *codexWebsocketSession, sessionID string, authID string, wsURL string, reason string, lastEvent string, err error) {
sessionStr := strings.TrimSpace(sessionID)
sessionKind := sessionObjectKind(sess)
terminalStatus := isTerminalEvent(lastEvent)
if err != nil {
if lastEvent != "" {
log.Infof("codex websockets: upstream disconnected session=%s auth=%s url=%s session_object=%s reason=%s last_event=%s is_terminal=%t err=%v", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind, strings.TrimSpace(reason), lastEvent, terminalStatus, err)
return
}
log.Infof("codex websockets: upstream disconnected session=%s auth=%s url=%s session_object=%s reason=%s is_terminal=false err=%v", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind, strings.TrimSpace(reason), err)
return
}
if lastEvent != "" {
log.Infof("codex websockets: upstream disconnected session=%s auth=%s url=%s session_object=%s reason=%s last_event=%s is_terminal=%t", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind, strings.TrimSpace(reason), lastEvent, terminalStatus)
return
}
log.Infof("codex websockets: upstream disconnected session=%s auth=%s url=%s session_object=%s reason=%s is_terminal=false", sessionStr, strings.TrimSpace(authID), strings.TrimSpace(wsURL), sessionKind, strings.TrimSpace(reason))
}
// CloseCodexWebsocketSessionsForAuthID closes all active Codex upstream websocket sessions
// associated with the supplied auth ID.
func CloseCodexWebsocketSessionsForAuthID(authID string, reason string) {
closeCodexWebsocketSessionsInStoreForAuthID(globalCodexWebsocketSessionStore, authID, reason)
}
func closeCodexWebsocketSessionsInStoreForAuthID(store *codexWebsocketSessionStore, authID string, reason string) {
authID = strings.TrimSpace(authID)
if authID == "" || store == nil {
return
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "auth_removed"
}
type sessionItem struct {
sessionID string
sess *codexWebsocketSession
}
store.mu.Lock()
items := make([]sessionItem, 0, len(store.sessions))
for sessionID, sess := range store.sessions {
items = append(items, sessionItem{sessionID: sessionID, sess: sess})
}
store.mu.Unlock()
matches := make([]sessionItem, 0)
for i := range items {
sess := items[i].sess
if sess == nil {
continue
}
sess.connMu.Lock()
sessAuthID := strings.TrimSpace(sess.authID)
sess.connMu.Unlock()
if sessAuthID == authID {
matches = append(matches, items[i])
}
}
if len(matches) == 0 {
return
}
toClose := make([]*codexWebsocketSession, 0, len(matches))
store.mu.Lock()
for i := range matches {
current, ok := store.sessions[matches[i].sessionID]
if !ok || current == nil || current != matches[i].sess {
continue
}
delete(store.sessions, matches[i].sessionID)
toClose = append(toClose, current)
}
store.mu.Unlock()
for i := range toClose {
closeCodexWebsocketSession(toClose[i], reason)
}
}