diff --git a/internal/runtime/executor/helps/codex_input_ids.go b/internal/runtime/executor/helps/codex_input_ids.go index 9a00b900c..14a63c13a 100644 --- a/internal/runtime/executor/helps/codex_input_ids.go +++ b/internal/runtime/executor/helps/codex_input_ids.go @@ -18,6 +18,9 @@ const ( codexFunctionCallItemIDPrefix = "fc" codexCustomToolCallItemIDPrefix = "ctc" codexCustomToolCallOutputItemIDPrefix = "ctco" + + codexInputItemIDOccupied uint8 = 1 << 0 + codexInputItemIDPreserved uint8 = 1 << 1 ) // SanitizeCodexInputItemIDs normalizes supported input item IDs for Codex, removes encrypted @@ -30,7 +33,7 @@ func SanitizeCodexInputItemIDs(body []byte) []byte { } items := input.Array() - occupied := make(map[string]struct{}, len(items)) + idStates := make(map[string]uint8, len(items)) for _, item := range items { if shouldDropCodexEncryptedReasoningItem(item) { continue @@ -39,13 +42,22 @@ func SanitizeCodexInputItemIDs(body []byte) []byte { if itemID.Type != gjson.String { continue } - id := normalizeCodexInputItemID(item, itemID.String()) + originalID := itemID.String() + id := normalizeCodexInputItemID(item, originalID) + state := idStates[id] + if id == originalID { + state |= codexInputItemIDPreserved + } if len([]rune(id)) <= codexInputItemIDLimit { - occupied[id] = struct{}{} + state |= codexInputItemIDOccupied + } + if state != 0 { + idStates[id] = state } } - mapped := make(map[string]string, len(items)) + var mapped map[string]string + var collisionMapped map[string]string rebuilt := make([]string, 0, len(items)) changed := false for _, item := range items { @@ -59,18 +71,39 @@ func SanitizeCodexInputItemIDs(body []byte) []byte { if itemID.Type == gjson.String { originalID := itemID.String() id := normalizeCodexInputItemID(item, originalID) + if id != originalID && idStates[id]&codexInputItemIDPreserved != 0 { + collisionID, ok := collisionMapped[id] + if !ok { + for attempt := 0; ; attempt++ { + collisionID = codexInputItemIDWithHashSuffix(id, attempt) + if idStates[collisionID]&codexInputItemIDOccupied != 0 { + continue + } + if collisionMapped == nil { + collisionMapped = make(map[string]string) + } + collisionMapped[id] = collisionID + idStates[collisionID] |= codexInputItemIDOccupied + break + } + } + id = collisionID + } if len([]rune(id)) > codexInputItemIDLimit { shortened, ok := mapped[id] if !ok { shortened = shortenCodexInputItemID(id) for attempt := 1; ; attempt++ { - if _, exists := occupied[shortened]; !exists { + if idStates[shortened]&codexInputItemIDOccupied == 0 { break } shortened = shortenCodexInputItemIDWithAttempt(id, attempt) } + if mapped == nil { + mapped = make(map[string]string) + } mapped[id] = shortened - occupied[shortened] = struct{}{} + idStates[shortened] |= codexInputItemIDOccupied } id = shortened } @@ -139,7 +172,14 @@ func shortenCodexInputItemIDWithAttempt(id string, attempt int) string { if len(runes) <= codexInputItemIDLimit { return id } + return codexInputItemIDWithHashSuffixRunes(id, runes, attempt) +} +func codexInputItemIDWithHashSuffix(id string, attempt int) string { + return codexInputItemIDWithHashSuffixRunes(id, []rune(id), attempt) +} + +func codexInputItemIDWithHashSuffixRunes(id string, runes []rune, attempt int) string { hashInput := id if attempt > 0 { hashInput += "\x00" + strconv.Itoa(attempt) @@ -147,5 +187,8 @@ func shortenCodexInputItemIDWithAttempt(id string, attempt int) string { sum := sha256.Sum256([]byte(hashInput)) suffix := "_" + hex.EncodeToString(sum[:8]) prefixLength := codexInputItemIDLimit - len(suffix) + if len(runes) < prefixLength { + prefixLength = len(runes) + } return string(runes[:prefixLength]) + suffix } diff --git a/internal/runtime/executor/helps/codex_input_ids_test.go b/internal/runtime/executor/helps/codex_input_ids_test.go index a6e2d519c..c6264f860 100644 --- a/internal/runtime/executor/helps/codex_input_ids_test.go +++ b/internal/runtime/executor/helps/codex_input_ids_test.go @@ -94,6 +94,71 @@ func TestSanitizeCodexInputItemIDsNormalizesResponseItemIDs(t *testing.T) { } } +func TestSanitizeCodexInputItemIDsAvoidsNormalizationCollisions(t *testing.T) { + for _, testCase := range []struct { + name string + itemType string + prefix string + }{ + {name: "message", itemType: "message", prefix: "msg_"}, + {name: "reasoning", itemType: "reasoning", prefix: "rs_"}, + {name: "function call", itemType: "function_call", prefix: "fc_"}, + {name: "custom tool call", itemType: "custom_tool_call", prefix: "ctc_"}, + {name: "custom tool call output", itemType: "custom_tool_call_output", prefix: "ctco_"}, + } { + for _, idCase := range []struct { + name string + invalidID string + }{ + {name: "short", invalidID: "item_collision"}, + {name: "overlong", invalidID: strings.Repeat("x", codexInputItemIDLimit-len([]rune(testCase.prefix))+1)}, + } { + prefixedID := testCase.prefix + idCase.invalidID + for _, order := range []struct { + name string + ids [2]string + prefixedIndex int + }{ + {name: "local first", ids: [2]string{idCase.invalidID, prefixedID}, prefixedIndex: 1}, + {name: "prefixed first", ids: [2]string{prefixedID, idCase.invalidID}, prefixedIndex: 0}, + } { + t.Run(testCase.name+"/"+idCase.name+"/"+order.name, func(t *testing.T) { + body := []byte(fmt.Sprintf(`{"input":[{"type":%q,"id":%q},{"type":%q,"id":%q}]}`, testCase.itemType, order.ids[0], testCase.itemType, order.ids[1])) + + first := SanitizeCodexInputItemIDs(body) + second := SanitizeCodexInputItemIDs(body) + normalizedAgain := SanitizeCodexInputItemIDs(first) + ids := [2]string{ + gjson.GetBytes(first, "input.0.id").String(), + gjson.GetBytes(first, "input.1.id").String(), + } + + if ids[0] == ids[1] { + t.Fatalf("distinct IDs collided after normalization: %q; payload=%s", ids[0], first) + } + for index, id := range ids { + if !strings.HasPrefix(id, testCase.prefix) { + t.Fatalf("input.%d.id = %q, want prefix %q", index, id, testCase.prefix) + } + if len([]rune(id)) > codexInputItemIDLimit { + t.Fatalf("input.%d.id length = %d, want at most %d: %q", index, len([]rune(id)), codexInputItemIDLimit, id) + } + } + if len([]rune(prefixedID)) <= codexInputItemIDLimit && ids[order.prefixedIndex] != prefixedID { + t.Fatalf("existing valid ID changed: got %q want %q", ids[order.prefixedIndex], prefixedID) + } + if string(first) != string(second) { + t.Fatalf("collision resolution is not deterministic: first=%s second=%s", first, second) + } + if string(first) != string(normalizedAgain) { + t.Fatalf("collision resolution is not idempotent: first=%s normalized_again=%s", first, normalizedAgain) + } + }) + } + } + } +} + func TestSanitizeCodexInputItemIDsNormalizesCustomToolCallIDs(t *testing.T) { const invalidID = "item_44e13caebc1ddf25f1337cbe" body := []byte(`{"input":[{"type":"custom_tool_call","id":"` + invalidID + `","call_id":"call-1","name":"lookup","input":"{}"}]}`) @@ -230,3 +295,24 @@ func BenchmarkSanitizeCodexInputItemIDsLargeNoopPayload(b *testing.B) { benchmarkSanitizeCodexInputItemIDsOutput = SanitizeCodexInputItemIDs(body) } } + +func BenchmarkSanitizeCodexInputItemIDsLargeHistory(b *testing.B) { + var payload strings.Builder + payload.Grow(64 << 10) + payload.WriteString(`{"input":[`) + for index := range 1000 { + if index > 0 { + payload.WriteByte(',') + } + fmt.Fprintf(&payload, `{"type":"message","id":"msg_%d","role":"user","content":"x"}`, index) + } + payload.WriteString(`]}`) + body := []byte(payload.String()) + + b.ReportAllocs() + b.SetBytes(int64(len(body))) + b.ResetTimer() + for b.Loop() { + benchmarkSanitizeCodexInputItemIDsOutput = SanitizeCodexInputItemIDs(body) + } +}