Files
CLIProxyAPI/internal/runtime/executor/helps/codex_input_ids.go
Luis Pater 6e6afab99a refactor: migrate gjson, sjson, and base64 to MoeForge/moego
- Replace tidwall/gjson and tidwall/sjson with github.com/MoeForge/moego/ojson
  and encoding/base64 with github.com/MoeForge/moego/base64 across the main
  module and examples
- Introduce cross-method *ojson.Doc request rewriting pipelines across
  thinking, signature, client/codex, runtime/executor/helps, pluginhost, and
  all provider executors with FinalizePayloadDoc as the final semantic barrier
- Convert all consecutive JSON mutations (>= 2 Set/SetRaw/Delete operations)
  to batch edits on ojson.Doc
- Upgrade read-only byte paths to ojson.GetBytesNoCopy,
  ojson.GetManyBytesNoCopy, and util.ParseGJSONBytesNoCopy with explicit
  strings.Clone ownership protection across streaming and persistent state
- Use base64.StdEncoding.ValidateString for zero-allocation signature checks
  and base64.EncodeDataURL for single-allocation data URL construction
2026-11-03 03:05:04 +08:00

237 lines
6.5 KiB
Go

package helps
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strconv"
"strings"
"github.com/MoeForge/moego/ojson"
)
const (
codexInputItemIDLimit = 64
codexMessageItemIDPrefix = "msg"
codexReasoningItemIDPrefix = "rs"
codexFunctionCallItemIDPrefix = "fc"
codexCustomToolCallItemIDPrefix = "ctc"
codexCustomToolCallOutputItemIDPrefix = "ctco"
codexToolSearchCallItemIDPrefix = "tsc"
codexInputItemIDOccupied uint8 = 1 << 0
codexInputItemIDPreserved uint8 = 1 << 1
codexInputItemIDSeen uint8 = 1 << 2
codexInputItemIDDuplicate uint8 = 1 << 3
)
// SanitizeCodexInputItemIDs normalizes supported input item IDs for Codex, removes encrypted
// reasoning items whose IDs exceed the Codex limit, and deterministically shortens
// other overlong input item IDs.
func SanitizeCodexInputItemIDs(body []byte) []byte {
if len(body) == 0 {
return body
}
doc := ojson.NewDoc(body)
if !SanitizeCodexInputItemIDsDoc(doc) {
return body
}
return doc.Bytes()
}
// SanitizeCodexInputItemIDsDoc normalizes supported input item IDs for Codex in place on doc,
// removes encrypted reasoning items whose IDs exceed the Codex limit, and deterministically
// shortens other overlong input item IDs.
func SanitizeCodexInputItemIDsDoc(doc *ojson.Doc) bool {
if doc == nil || doc.Len() == 0 {
return false
}
input := doc.Get("input")
if !input.IsArray() {
return false
}
items := input.Array()
idStates := make(map[string]uint8, len(items))
hasDropped := false
for _, item := range items {
if shouldDropCodexEncryptedReasoningItem(item) {
hasDropped = true
continue
}
itemID := item.Get("id")
if itemID.Type != ojson.String {
continue
}
originalID := itemID.String()
id := normalizeCodexInputItemID(item, originalID)
state := idStates[id]
if state&codexInputItemIDSeen != 0 {
state |= codexInputItemIDDuplicate
}
state |= codexInputItemIDSeen
if id == originalID {
state |= codexInputItemIDPreserved
}
if len([]rune(id)) <= codexInputItemIDLimit {
state |= codexInputItemIDOccupied
}
idStates[id] = state
}
var mapped map[string]string
var collisionMapped map[string]string
var dropped map[int]bool
if hasDropped {
dropped = make(map[int]bool)
}
changed := false
for idx, item := range items {
if shouldDropCodexEncryptedReasoningItem(item) {
dropped[idx] = true
changed = true
continue
}
itemID := item.Get("id")
if itemID.Type == ojson.String {
originalID := itemID.String()
id := normalizeCodexInputItemID(item, originalID)
if id != originalID && idStates[id]&(codexInputItemIDPreserved|codexInputItemIDDuplicate) != 0 {
collisionKey := id + "\x00" + originalID
if collisionID, ok := collisionMapped[collisionKey]; ok {
id = collisionID
} else if idStates[id]&codexInputItemIDPreserved == 0 {
if collisionMapped == nil {
collisionMapped = make(map[string]string)
}
collisionMapped[collisionKey] = id
idStates[id] |= codexInputItemIDPreserved
} else {
var collisionID string
for attempt := 0; ; attempt++ {
collisionID = codexInputItemIDWithHashSuffix(id, attempt)
if idStates[collisionID]&codexInputItemIDOccupied != 0 {
continue
}
if collisionMapped == nil {
collisionMapped = make(map[string]string)
}
collisionMapped[collisionKey] = collisionID
idStates[collisionID] |= codexInputItemIDOccupied | codexInputItemIDPreserved
break
}
id = collisionID
}
}
if len([]rune(id)) > codexInputItemIDLimit {
shortened, ok := mapped[id]
if !ok {
shortened = shortenCodexInputItemID(id)
for attempt := 1; ; attempt++ {
if idStates[shortened]&codexInputItemIDOccupied == 0 {
break
}
shortened = shortenCodexInputItemIDWithAttempt(id, attempt)
}
if mapped == nil {
mapped = make(map[string]string)
}
mapped[id] = shortened
idStates[shortened] |= codexInputItemIDOccupied
}
id = shortened
}
if id != originalID {
if errSet := doc.Set(fmt.Sprintf("input.%d.id", idx), id); errSet == nil {
changed = true
}
}
}
}
if !changed {
return false
}
if hasDropped {
updatedItems := doc.Get("input").Array()
rebuilt := make([][]byte, 0, len(updatedItems)-len(dropped))
for idx, item := range updatedItems {
if !dropped[idx] {
rebuilt = append(rebuilt, []byte(item.Raw))
}
}
return doc.SetRawBytes("input", JoinRawJSONArray(rebuilt)) == nil
}
return true
}
func normalizeCodexInputItemID(item ojson.Result, id string) string {
var prefix string
switch item.Get("type").String() {
case "message":
prefix = codexMessageItemIDPrefix
case "reasoning":
prefix = codexReasoningItemIDPrefix
case "function_call":
prefix = codexFunctionCallItemIDPrefix
case "custom_tool_call":
prefix = codexCustomToolCallItemIDPrefix
case "custom_tool_call_output":
prefix = codexCustomToolCallOutputItemIDPrefix
case "tool_search_call":
if after, ok := strings.CutPrefix(id, "ts_"); ok && after != "" {
return codexToolSearchCallItemIDPrefix + "_" + after
}
prefix = codexToolSearchCallItemIDPrefix
default:
return id
}
if id == "" || strings.HasPrefix(id, prefix) {
return id
}
return prefix + "_" + id
}
func shouldDropCodexEncryptedReasoningItem(item ojson.Result) bool {
if item.Get("type").String() != "reasoning" {
return false
}
itemID := item.Get("id")
if itemID.Type != ojson.String || len([]rune(itemID.String())) <= codexInputItemIDLimit {
return false
}
encryptedContent := item.Get("encrypted_content")
return encryptedContent.Type == ojson.String && encryptedContent.String() != ""
}
func shortenCodexInputItemID(id string) string {
return shortenCodexInputItemIDWithAttempt(id, 0)
}
func shortenCodexInputItemIDWithAttempt(id string, attempt int) string {
runes := []rune(id)
if len(runes) <= codexInputItemIDLimit {
return id
}
return codexInputItemIDWithHashSuffixRunes(id, runes, attempt)
}
func codexInputItemIDWithHashSuffix(id string, attempt int) string {
return codexInputItemIDWithHashSuffixRunes(id, []rune(id), attempt)
}
func codexInputItemIDWithHashSuffixRunes(id string, runes []rune, attempt int) string {
hashInput := id
if attempt > 0 {
hashInput += "\x00" + strconv.Itoa(attempt)
}
sum := sha256.Sum256([]byte(hashInput))
suffix := "_" + hex.EncodeToString(sum[:8])
prefixLength := codexInputItemIDLimit - len(suffix)
if len(runes) < prefixLength {
prefixLength = len(runes)
}
return string(runes[:prefixLength]) + suffix
}