From e56abd56f1424b9b25b4609839833724100f3ef5 Mon Sep 17 00:00:00 2001 From: sususu Date: Wed, 9 Sep 2026 17:49:13 +0800 Subject: [PATCH 1/2] fix(translator): strip unsupported unicode property escape patterns from tool schemas - Add HasUnsupportedUnicodePropertyEscape in internal/util to detect \p{...} / \P{...} escapes that fail Python re compilation. - Strip incompatible pattern attributes during tool parameter normalization in codex/claude and openai/claude translators. - Provide schema-aware fallback stripping in codex executor helps to protect downstream Codex requests without mutating non-schema user data. - Export unified schema keyword lists in internal/util to eliminate duplication. - Add comprehensive unit tests covering Artifact fixtures, lookaheads, and user data preservation. Closes: #5644 --- .../executor/helps/codex_tool_schema.go | 90 +++++++- .../executor/helps/codex_tool_schema_test.go | 198 ++++++++++++++++++ .../codex/claude/codex_claude_request.go | 44 ++-- .../codex/claude/codex_claude_request_test.go | 102 +++++++++ .../openai/claude/openai_claude_request.go | 25 ++- .../claude/openai_claude_request_test.go | 102 +++++++++ internal/util/claude_schema.go | 49 +++++ internal/util/claude_schema_test.go | 78 +++++++ 8 files changed, 653 insertions(+), 35 deletions(-) diff --git a/internal/runtime/executor/helps/codex_tool_schema.go b/internal/runtime/executor/helps/codex_tool_schema.go index 5b56bd8e3..b978e153b 100644 --- a/internal/runtime/executor/helps/codex_tool_schema.go +++ b/internal/runtime/executor/helps/codex_tool_schema.go @@ -1,6 +1,9 @@ package helps import ( + "bytes" + "encoding/json" + "io" "math/big" "strconv" "strings" @@ -8,6 +11,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/tidwall/sjson" + + "github.com/router-for-me/CLIProxyAPI/v7/internal/util" ) const ( @@ -89,7 +94,7 @@ func normalizeCodexTool(tool gjson.Result) ([]byte, bool) { return nil, false } - log.Debugf("codex: simplified complex schema unions for tool %s to avoid upstream abort", tool.Get("name").String()) + log.Debugf("codex: normalized schema for tool %s to avoid upstream abort", tool.Get("name").String()) return updatedTool, true } @@ -97,6 +102,12 @@ func normalizeCodexParameters(params gjson.Result) ([]byte, bool) { rawParams := []byte(params.Raw) changed := false + if sanitizedParams, patternChanged := stripIncompatiblePatternsFromJSON(rawParams); patternChanged { + rawParams = sanitizedParams + changed = true + params = gjson.ParseBytes(rawParams) + } + properties := params.Get("properties") if properties.Exists() && properties.IsObject() { for propName, propVal := range properties.Map() { @@ -115,6 +126,83 @@ func normalizeCodexParameters(params gjson.Result) ([]byte, bool) { return rawParams, changed } +// stripIncompatiblePatternsFromJSON recursively removes pattern attributes containing +// unsupported Unicode property escapes (\p{...} / \P{...}) from parameter schemas. +// It is schema-aware: only subschemas under known JSON Schema keyword locations are visited, +// preventing accidental deletion of 'pattern' keys inside user data (e.g. description, default, enum). +func stripIncompatiblePatternsFromJSON(raw []byte) ([]byte, bool) { + rawStr := string(raw) + if !strings.Contains(rawStr, `\p{`) && !strings.Contains(rawStr, `\P{`) { + return raw, false + } + var root any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&root); err != nil || root == nil { + return raw, false + } + // Verify no trailing garbage + var dummy any + if err := dec.Decode(&dummy); err != io.EOF { + return raw, false + } + if !stripIncompatiblePatterns(root) { + return raw, false + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(root); err != nil { + return raw, false + } + return bytes.TrimSpace(buf.Bytes()), true +} + +func stripIncompatiblePatterns(v any) bool { + changed := false + switch schema := v.(type) { + case map[string]any: + if patternVal, ok := schema["pattern"].(string); ok && util.HasUnsupportedUnicodePropertyEscape(patternVal) { + delete(schema, "pattern") + changed = true + } + + for _, mapKey := range util.SchemaMapKeywords { + if subMap, ok := schema[mapKey].(map[string]any); ok { + for _, subSchema := range subMap { + if stripIncompatiblePatterns(subSchema) { + changed = true + } + } + } + } + + for _, valKey := range util.SchemaValueKeywords { + if val, exists := schema[valKey]; exists { + switch sub := val.(type) { + case map[string]any: + if stripIncompatiblePatterns(sub) { + changed = true + } + case []any: + for _, item := range sub { + if stripIncompatiblePatterns(item) { + changed = true + } + } + } + } + } + case []any: + for _, item := range schema { + if stripIncompatiblePatterns(item) { + changed = true + } + } + } + return changed +} + func normalizeCodexPropertySchema(prop gjson.Result) ([]byte, bool) { if !prop.IsObject() { return nil, false diff --git a/internal/runtime/executor/helps/codex_tool_schema_test.go b/internal/runtime/executor/helps/codex_tool_schema_test.go index 74983ec05..ba4a834f7 100644 --- a/internal/runtime/executor/helps/codex_tool_schema_test.go +++ b/internal/runtime/executor/helps/codex_tool_schema_test.go @@ -491,3 +491,201 @@ func TestNormalizeCodexToolSchemas_NamespaceToolSimplified(t *testing.T) { t.Fatalf("expected action.enum with 8 items") } } + +func TestNormalizeCodexToolSchemas_StripsUnsupportedUnicodePropertyEscapePatterns(t *testing.T) { + input := []byte(`{ + "model": "gpt-5.6", + "tools": [{ + "type": "function", + "name": "Artifact", + "parameters": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "field to edit", + "pattern": "^(?!__.*__$)[^\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\"\\\\./[\\]]{1,200}$" + }, + "asset_id": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + } + }, + "required": ["field"] + } + }] + }`) + + out := NormalizeCodexToolSchemas(input) + + tool := gjson.GetBytes(out, "tools.0") + params := tool.Get("parameters") + + // Unsupported pattern must be removed + if params.Get("properties.field.pattern").Exists() { + t.Errorf("expected properties.field.pattern to be removed, got: %s", params.Get("properties.field.pattern").Raw) + } + if got := params.Get("properties.field.type").String(); got != "string" { + t.Errorf("expected properties.field.type == 'string', got %q", got) + } + + // Valid pattern must be preserved + if got := params.Get("properties.asset_id.pattern").String(); got != "^[0-9a-f]{32}$" { + t.Errorf("expected properties.asset_id.pattern preserved, got %q", got) + } + + // Idempotence test + outAgain := NormalizeCodexToolSchemas(out) + if string(outAgain) != string(out) { + t.Errorf("expected NormalizeCodexToolSchemas to be idempotent") + } +} + +func TestNormalizeCodexToolSchemas_PreservesNonSchemaPatternKeys(t *testing.T) { + // A property whose default, enum, or description metadata contains a nested object + // with a 'pattern' key must NOT be mutated, because it is user data, not a schema. + input := []byte(`{ + "model": "gpt-5.6", + "tools": [{ + "type": "function", + "name": "config_tool", + "parameters": { + "type": "object", + "properties": { + "regex_config": { + "type": "object", + "default": { + "pattern": "\\p{L}+" + }, + "enum": [ + {"pattern": "\\p{N}+"} + ] + }, + "real_schema": { + "type": "string", + "pattern": "\\p{L}+" + } + } + } + }] + }`) + + out := NormalizeCodexToolSchemas(input) + + tool := gjson.GetBytes(out, "tools.0") + params := tool.Get("parameters") + + // Real schema pattern must be removed + if params.Get("properties.real_schema.pattern").Exists() { + t.Errorf("expected real_schema.pattern to be removed, got: %s", params.Get("properties.real_schema").Raw) + } + + // User data under default and enum must be PRESERVED + if got := params.Get("properties.regex_config.default.pattern").String(); got != `\p{L}+` { + t.Errorf("expected default.pattern preserved, got %q", got) + } + if got := params.Get("properties.regex_config.enum.0.pattern").String(); got != `\p{N}+` { + t.Errorf("expected enum.0.pattern preserved, got %q", got) + } +} + +func TestNormalizeCodexToolSchemas_CoversAllSchemaKeywordLocations(t *testing.T) { + input := []byte(`{ + "model": "gpt-5.6", + "tools": [{ + "type": "function", + "name": "deep_tool", + "parameters": { + "type": "object", + "$defs": { + "custom_type": { + "type": "string", + "pattern": "\\p{L}+" + } + }, + "additionalProperties": { + "type": "string", + "pattern": "\\p{N}+" + }, + "patternProperties": { + "^s_": { + "type": "string", + "pattern": "\\p{M}+" + } + }, + "if": { + "properties": { + "flag": { + "type": "string", + "pattern": "\\p{P}+" + } + } + }, + "then": { + "properties": { + "val": { + "type": "string", + "pattern": "\\p{S}+" + } + } + }, + "else": { + "properties": { + "other": { + "type": "string", + "pattern": "\\p{Z}+" + } + } + } + } + }] + }`) + + out := NormalizeCodexToolSchemas(input) + + tool := gjson.GetBytes(out, "tools.0") + params := tool.Get("parameters") + + // All subschemas in schema-aware locations must have their incompatible patterns stripped + if params.Get("$defs.custom_type.pattern").Exists() { + t.Errorf("expected $defs.custom_type.pattern to be removed") + } + if params.Get("additionalProperties.pattern").Exists() { + t.Errorf("expected additionalProperties.pattern to be removed") + } + if params.Get("patternProperties.^s_.pattern").Exists() { + t.Errorf("expected patternProperties.^s_.pattern to be removed") + } + if params.Get("if.properties.flag.pattern").Exists() { + t.Errorf("expected if.properties.flag.pattern to be removed") + } + if params.Get("then.properties.val.pattern").Exists() { + t.Errorf("expected then.properties.val.pattern to be removed") + } + if params.Get("else.properties.other.pattern").Exists() { + t.Errorf("expected else.properties.other.pattern to be removed") + } + + // Subschema types must be preserved + if got := params.Get("$defs.custom_type.type").String(); got != "string" { + t.Errorf("expected $defs.custom_type.type == 'string', got %q", got) + } +} + +func TestNormalizeCodexToolSchemas_MalformedOrEmptyParametersFallback(t *testing.T) { + // Malformed JSON, non-object parameters, null, and empty payloads must not panic + cases := [][]byte{ + []byte(`{"model":"gpt-5.6","tools":[{"type":"function","name":"t","parameters":null}]}`), + []byte(`{"model":"gpt-5.6","tools":[{"type":"function","name":"t","parameters":"not_an_object"}]}`), + []byte(`{"model":"gpt-5.6","tools":[{"type":"function","name":"t","parameters":{"type":"object"}}]}`), + []byte(`{"model":"gpt-5.6","tools":[]}`), + []byte(`{"model":"gpt-5.6"}`), + } + + for i, c := range cases { + out := NormalizeCodexToolSchemas(c) + if len(out) == 0 { + t.Errorf("case %d: unexpected empty output", i) + } + } +} diff --git a/internal/translator/codex/claude/codex_claude_request.go b/internal/translator/codex/claude/codex_claude_request.go index 5e883c37b..b9a8a297e 100644 --- a/internal/translator/codex/claude/codex_claude_request.go +++ b/internal/translator/codex/claude/codex_claude_request.go @@ -665,8 +665,9 @@ func buildReverseMapFromClaudeOriginalToShort(original []byte) map[string]string return m } -// normalizeToolParameters ensures object schemas contain at least an empty properties map -// and strips dialect keywords ($schema, $id) from schema objects. +// normalizeToolParameters ensures object schemas contain at least an empty properties map, +// strips dialect keywords ($schema, $id), and drops regex patterns containing unsupported +// Unicode property escapes (\p{...} / \P{...}) that cause upstream schema validation failures. func normalizeToolParameters(raw string) string { raw = strings.TrimSpace(raw) if raw == "" || raw == "null" || !gjson.Valid(raw) { @@ -721,6 +722,9 @@ func stripDialectKeywordsFromSchema(v any) { case map[string]any: delete(schema, "$schema") delete(schema, "$id") + if patternVal, ok := schema["pattern"].(string); ok && util.HasUnsupportedUnicodePropertyEscape(patternVal) { + delete(schema, "pattern") + } for _, mapKey := range codexSchemaMapKeywords { if subMap, ok := schema[mapKey].(map[string]any); ok { @@ -749,36 +753,12 @@ func stripDialectKeywordsFromSchema(v any) { } } -// codexSchemaMapKeywords holds JSON Schema keywords whose values are maps of -// subschemas; codexSchemaValueKeywords holds keywords with a single nested -// schema or a list of schemas. -var codexSchemaMapKeywords = [...]string{ - "properties", - "$defs", - "definitions", - "patternProperties", - "dependentSchemas", - "dependencies", -} - -var codexSchemaValueKeywords = [...]string{ - "items", - "prefixItems", - "contains", - "additionalProperties", - "propertyNames", - "unevaluatedProperties", - "unevaluatedItems", - "additionalItems", - "contentSchema", - "anyOf", - "oneOf", - "allOf", - "not", - "if", - "then", - "else", -} +// codexSchemaMapKeywords and codexSchemaValueKeywords reference the unified JSON Schema keywords +// declared in internal/util. +var ( + codexSchemaMapKeywords = util.SchemaMapKeywords + codexSchemaValueKeywords = util.SchemaValueKeywords +) // codexSchemaMissesRequired reports whether a JSON Schema has any declared // property missing from its sibling required list (recursively). OpenAI diff --git a/internal/translator/codex/claude/codex_claude_request_test.go b/internal/translator/codex/claude/codex_claude_request_test.go index 931ad3d94..bd7f52d43 100644 --- a/internal/translator/codex/claude/codex_claude_request_test.go +++ b/internal/translator/codex/claude/codex_claude_request_test.go @@ -1124,3 +1124,105 @@ func TestConvertClaudeRequestToCodex_StripsNestedToolSchemaMeta(t *testing.T) { t.Errorf("expected parameters.$defs.hint.$id to be removed, got %v", params.Get("$defs.hint.$id").Raw) } } + +func TestConvertClaudeRequestToCodex_StripsUnsupportedUnicodePropertyEscapePatterns(t *testing.T) { + inputJSON := `{ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{ + "name": "Artifact", + "description": "Render an HTML file to an Artifact", + "input_schema": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "field to replace", + "pattern": "^(?!__.*__$)[^\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\"\\\\./[\\]]{1,200}$" + }, + "asset_id": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "lookahead_safe": { + "type": "string", + "pattern": "^(?!__.*__$).{1,200}$" + }, + "literal_p": { + "type": "string", + "pattern": "^\\\\p{Cc}$" + }, + "nested": { + "type": "object", + "properties": { + "inner_field": { + "type": "string", + "pattern": "\\P{L}+" + } + } + }, + "union_field": { + "anyOf": [ + { + "type": "string", + "pattern": "\\p{N}+" + }, + { + "type": "null" + } + ] + } + }, + "required": ["field"] + } + }] + }` + + translated := ConvertClaudeRequestToCodex("gpt-5.6", []byte(inputJSON), false) + tools := gjson.GetBytes(translated, "tools").Array() + if len(tools) == 0 { + t.Fatalf("expected tools in translated payload, got: %s", translated) + } + params := tools[0].Get("parameters") + + // The \p{...} pattern on field must be removed to avoid Python re failure upstream + if params.Get("properties.field.pattern").Exists() { + t.Errorf("expected properties.field.pattern to be removed, got %v", params.Get("properties.field.pattern").Raw) + } + // Other attributes on field must be preserved + if got := params.Get("properties.field.type").String(); got != "string" { + t.Errorf("expected properties.field.type == 'string', got %q", got) + } + if got := params.Get("properties.field.description").String(); got != "field to replace" { + t.Errorf("expected properties.field.description == 'field to replace', got %q", got) + } + + // Valid patterns without unescaped \p must be preserved + if got := params.Get("properties.asset_id.pattern").String(); got != "^[0-9a-f]{32}$" { + t.Errorf("expected asset_id.pattern preserved, got %q", got) + } + if got := params.Get("properties.lookahead_safe.pattern").String(); got != "^(?!__.*__$).{1,200}$" { + t.Errorf("expected lookahead_safe.pattern preserved, got %q", got) + } + if got := params.Get("properties.literal_p.pattern").String(); got != "^\\\\p{Cc}$" { + t.Errorf("expected literal_p.pattern preserved, got %q", got) + } + + // Nested object with \P{L}+ must have its pattern removed + if params.Get("properties.nested.properties.inner_field.pattern").Exists() { + t.Errorf("expected properties.nested.properties.inner_field.pattern to be removed, got %v", params.Get("properties.nested.properties.inner_field.pattern").Raw) + } + if got := params.Get("properties.nested.properties.inner_field.type").String(); got != "string" { + t.Errorf("expected nested inner_field.type preserved, got %q", got) + } + + // Union anyOf with \p{N}+ must have its pattern removed + if params.Get("properties.union_field.anyOf.0.pattern").Exists() { + t.Errorf("expected union_field.anyOf.0.pattern to be removed, got %v", params.Get("properties.union_field.anyOf.0.pattern").Raw) + } + + // Required array must be preserved + if got := params.Get("required.0").String(); got != "field" { + t.Errorf("expected required.0 == 'field', got %q", got) + } +} diff --git a/internal/translator/openai/claude/openai_claude_request.go b/internal/translator/openai/claude/openai_claude_request.go index e28679d49..b4ae3d7a4 100644 --- a/internal/translator/openai/claude/openai_claude_request.go +++ b/internal/translator/openai/claude/openai_claude_request.go @@ -388,8 +388,29 @@ func normalizeObjectSchemaProperties(schema any) any { value["properties"] = map[string]any{} } } - for key, child := range value { - value[key] = normalizeObjectSchemaProperties(child) + if patternVal, ok := value["pattern"].(string); ok && util.HasUnsupportedUnicodePropertyEscape(patternVal) { + delete(value, "pattern") + } + + for _, mapKey := range util.SchemaMapKeywords { + if subMap, ok := value[mapKey].(map[string]any); ok { + for subKey, subSchema := range subMap { + subMap[subKey] = normalizeObjectSchemaProperties(subSchema) + } + } + } + + for _, valKey := range util.SchemaValueKeywords { + if val, exists := value[valKey]; exists { + switch sub := val.(type) { + case map[string]any: + value[valKey] = normalizeObjectSchemaProperties(sub) + case []any: + for i, item := range sub { + sub[i] = normalizeObjectSchemaProperties(item) + } + } + } } return value case []any: diff --git a/internal/translator/openai/claude/openai_claude_request_test.go b/internal/translator/openai/claude/openai_claude_request_test.go index 639f7f80d..9515229de 100644 --- a/internal/translator/openai/claude/openai_claude_request_test.go +++ b/internal/translator/openai/claude/openai_claude_request_test.go @@ -1017,3 +1017,105 @@ func TestConvertClaudeRequestToOpenAI_ToolWithoutInputSchemaDefaultsParameters(t } } } + +func TestConvertClaudeRequestToOpenAI_StripsUnsupportedUnicodePropertyEscapePatterns(t *testing.T) { + inputJSON := []byte(`{ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{ + "name": "Artifact", + "description": "Render an HTML file to an Artifact", + "input_schema": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "field to replace", + "pattern": "^(?!__.*__$)[^\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\"\\\\./[\\]]{1,200}$" + }, + "asset_id": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "lookahead_safe": { + "type": "string", + "pattern": "^(?!__.*__$).{1,200}$" + } + } + } + }] + }`) + + output := ConvertClaudeRequestToOpenAI("gpt-5.6", inputJSON, false) + outputJSON := gjson.ParseBytes(output) + + params := outputJSON.Get("tools.0.function.parameters") + if !params.Exists() { + t.Fatalf("expected function.parameters in output: %s", output) + } + + // Incompatible \p{...} pattern must be stripped + if params.Get("properties.field.pattern").Exists() { + t.Errorf("expected properties.field.pattern to be removed, got: %s", params.Get("properties.field.pattern").Raw) + } + if got := params.Get("properties.field.type").String(); got != "string" { + t.Errorf("expected properties.field.type == 'string', got %q", got) + } + + // Valid patterns must remain intact + if got := params.Get("properties.asset_id.pattern").String(); got != "^[0-9a-f]{32}$" { + t.Errorf("expected properties.asset_id.pattern preserved, got %q", got) + } + if got := params.Get("properties.lookahead_safe.pattern").String(); got != "^(?!__.*__$).{1,200}$" { + t.Errorf("expected properties.lookahead_safe.pattern preserved, got %q", got) + } +} + +func TestConvertClaudeRequestToOpenAI_PreservesNonSchemaPatternKeys(t *testing.T) { + inputJSON := []byte(`{ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{ + "name": "config_tool", + "input_schema": { + "type": "object", + "properties": { + "regex_config": { + "type": "object", + "default": { + "pattern": "\\p{L}+" + }, + "enum": [ + {"pattern": "\\p{N}+"} + ] + }, + "real_schema": { + "type": "string", + "pattern": "\\p{L}+" + } + } + } + }] + }`) + + output := ConvertClaudeRequestToOpenAI("gpt-5.6", inputJSON, false) + outputJSON := gjson.ParseBytes(output) + + params := outputJSON.Get("tools.0.function.parameters") + if !params.Exists() { + t.Fatalf("expected function.parameters in output: %s", output) + } + + // Real schema pattern must be removed + if params.Get("properties.real_schema.pattern").Exists() { + t.Errorf("expected real_schema.pattern to be removed, got: %s", params.Get("properties.real_schema").Raw) + } + + // User data under default and enum must be PRESERVED + if got := params.Get("properties.regex_config.default.pattern").String(); got != `\p{L}+` { + t.Errorf("expected default.pattern preserved, got %q", got) + } + if got := params.Get("properties.regex_config.enum.0.pattern").String(); got != `\p{N}+` { + t.Errorf("expected enum.0.pattern preserved, got %q", got) + } +} diff --git a/internal/util/claude_schema.go b/internal/util/claude_schema.go index c8ea0a7d2..45a59f77d 100644 --- a/internal/util/claude_schema.go +++ b/internal/util/claude_schema.go @@ -120,3 +120,52 @@ func mergeClaudeSchemaRequired(root map[string]json.RawMessage, branchRequired j root["required"] = requiredRaw } } + +// HasUnsupportedUnicodePropertyEscape reports whether a regular expression string +// contains unescaped Unicode property escape sequences (\p{...} or \P{...}). +// Python's built-in re module (and schema validators relying on it) fails compilation +// with "bad escape \p" on these sequences. +func HasUnsupportedUnicodePropertyEscape(pattern string) bool { + for i := 0; i < len(pattern); i++ { + if pattern[i] != '\\' { + continue + } + if i+2 < len(pattern) && + (pattern[i+1] == 'p' || pattern[i+1] == 'P') && + pattern[i+2] == '{' { + return true + } + i++ // skip the escaped character (including escaped backslash) + } + return false +} + +// SchemaMapKeywords lists JSON Schema keywords whose values are maps of subschemas. +var SchemaMapKeywords = [...]string{ + "properties", + "$defs", + "definitions", + "patternProperties", + "dependentSchemas", + "dependencies", +} + +// SchemaValueKeywords lists JSON Schema keywords with a single nested subschema or a slice of subschemas. +var SchemaValueKeywords = [...]string{ + "items", + "prefixItems", + "contains", + "additionalProperties", + "propertyNames", + "unevaluatedProperties", + "unevaluatedItems", + "additionalItems", + "contentSchema", + "anyOf", + "oneOf", + "allOf", + "not", + "if", + "then", + "else", +} diff --git a/internal/util/claude_schema_test.go b/internal/util/claude_schema_test.go index b10836c1f..6730cdc66 100644 --- a/internal/util/claude_schema_test.go +++ b/internal/util/claude_schema_test.go @@ -112,3 +112,81 @@ func TestNormalizeClaudeToolInputSchema(t *testing.T) { }) } } + +func TestHasUnsupportedUnicodePropertyEscape(t *testing.T) { + tests := []struct { + name string + pattern string + want bool + }{ + { + name: "Artifact regex with multiple \\p escapes", + pattern: `^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\./[\]]{1,200}$`, + want: true, + }, + { + name: "Uppercase \\P property escape", + pattern: `^\P{L}+$`, + want: true, + }, + { + name: "Escaped backslash before p is literal and safe", + pattern: `^\\p{Cc}$`, + want: false, + }, + { + name: "Three backslashes means literal backslash plus real \\p escape", + pattern: `^\\\p{Cc}$`, + want: true, + }, + { + name: "Four backslashes means two literal backslashes", + pattern: `^\\\\p{Cc}$`, + want: false, + }, + { + name: "Standard hex character pattern", + pattern: `^[0-9a-f]{32}$`, + want: false, + }, + { + name: "Negative lookahead without unicode properties", + pattern: `^(?!__.*__$).{1,200}$`, + want: false, + }, + { + name: "Backreference safe in Python re", + pattern: `^(a)\1$`, + want: false, + }, + { + name: "Trailing single backslash", + pattern: `abc\`, + want: false, + }, + { + name: "Backslash followed by p without brace", + pattern: `\p`, + want: false, + }, + { + name: "Backslash followed by p and brace without close", + pattern: `\p{`, + want: true, + }, + { + name: "Empty string", + pattern: ``, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := HasUnsupportedUnicodePropertyEscape(tt.pattern) + if got != tt.want { + t.Errorf("HasUnsupportedUnicodePropertyEscape(%q) = %v, want %v", tt.pattern, got, tt.want) + } + }) + } +} From 37ce368c5002bc4a83cd622ec01602b0b7984672 Mon Sep 17 00:00:00 2001 From: sususu Date: Wed, 9 Sep 2026 18:03:39 +0800 Subject: [PATCH 2/2] fix(schema): inspect patternProperties keys and avoid Unicode escape fast-path bypass - Check for \u in fast-path check to prevent JSON Unicode escapes from bypassing inspection. - Inspect regex keys under patternProperties and drop keys with unsupported Unicode property escapes. - Add tests covering Unicode escape representations (\u005c, \u0070, \u0050) and patternProperties keys. --- .../executor/helps/codex_tool_schema.go | 17 +++- .../executor/helps/codex_tool_schema_test.go | 85 +++++++++++++++++++ .../codex/claude/codex_claude_request.go | 14 +++ .../codex/claude/codex_claude_request_test.go | 36 ++++++++ .../openai/claude/openai_claude_request.go | 14 +++ .../claude/openai_claude_request_test.go | 37 ++++++++ 6 files changed, 202 insertions(+), 1 deletion(-) diff --git a/internal/runtime/executor/helps/codex_tool_schema.go b/internal/runtime/executor/helps/codex_tool_schema.go index b978e153b..2bc1dd863 100644 --- a/internal/runtime/executor/helps/codex_tool_schema.go +++ b/internal/runtime/executor/helps/codex_tool_schema.go @@ -132,7 +132,7 @@ func normalizeCodexParameters(params gjson.Result) ([]byte, bool) { // preventing accidental deletion of 'pattern' keys inside user data (e.g. description, default, enum). func stripIncompatiblePatternsFromJSON(raw []byte) ([]byte, bool) { rawStr := string(raw) - if !strings.Contains(rawStr, `\p{`) && !strings.Contains(rawStr, `\P{`) { + if !strings.Contains(rawStr, `\p{`) && !strings.Contains(rawStr, `\P{`) && !strings.Contains(rawStr, `\u`) { return raw, false } var root any @@ -167,7 +167,22 @@ func stripIncompatiblePatterns(v any) bool { changed = true } + // Inspect regex keys under patternProperties + if patternProps, ok := schema["patternProperties"].(map[string]any); ok { + for patternKey, subSchema := range patternProps { + if util.HasUnsupportedUnicodePropertyEscape(patternKey) { + delete(patternProps, patternKey) + changed = true + } else if stripIncompatiblePatterns(subSchema) { + changed = true + } + } + } + for _, mapKey := range util.SchemaMapKeywords { + if mapKey == "patternProperties" { + continue + } if subMap, ok := schema[mapKey].(map[string]any); ok { for _, subSchema := range subMap { if stripIncompatiblePatterns(subSchema) { diff --git a/internal/runtime/executor/helps/codex_tool_schema_test.go b/internal/runtime/executor/helps/codex_tool_schema_test.go index ba4a834f7..769badb4d 100644 --- a/internal/runtime/executor/helps/codex_tool_schema_test.go +++ b/internal/runtime/executor/helps/codex_tool_schema_test.go @@ -689,3 +689,88 @@ func TestNormalizeCodexToolSchemas_MalformedOrEmptyParametersFallback(t *testing } } } + +func TestNormalizeCodexToolSchemas_JSONUnicodeEscapeBypassPrevention(t *testing.T) { + // Patterns encoded using JSON Unicode escapes (e.g. \u005c for '\' or \u0070 for 'p') + // decode to \p{...} / \P{...} and must not be skipped by the fast-path check. + input := []byte(`{ + "model": "gpt-5.6", + "tools": [{ + "type": "function", + "name": "escape_bypass_tool", + "parameters": { + "type": "object", + "properties": { + "p1": { + "type": "string", + "pattern": "\u005c\u0070{L}+" + }, + "p2": { + "type": "string", + "pattern": "\u005cp{Cc}" + }, + "p3": { + "type": "string", + "pattern": "\u005c\u0050{N}+" + }, + "valid": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + } + } + } + }] + }`) + + out := NormalizeCodexToolSchemas(input) + tool := gjson.GetBytes(out, "tools.0") + params := tool.Get("parameters") + + if params.Get("properties.p1.pattern").Exists() { + t.Errorf("expected properties.p1.pattern (\\u0070) to be removed, got: %s", params.Get("properties.p1.pattern").Raw) + } + if params.Get("properties.p2.pattern").Exists() { + t.Errorf("expected properties.p2.pattern (\\u005c) to be removed, got: %s", params.Get("properties.p2.pattern").Raw) + } + if params.Get("properties.p3.pattern").Exists() { + t.Errorf("expected properties.p3.pattern (\\u0050) to be removed, got: %s", params.Get("properties.p3.pattern").Raw) + } + if got := params.Get("properties.valid.pattern").String(); got != "^[0-9a-f]{32}$" { + t.Errorf("expected valid.pattern to be preserved, got %q", got) + } +} + +func TestNormalizeCodexToolSchemas_PatternPropertiesKeySanitization(t *testing.T) { + input := []byte(`{ + "model": "gpt-5.6", + "tools": [{ + "type": "function", + "name": "pattern_props_tool", + "parameters": { + "type": "object", + "patternProperties": { + "^\\\\p{L}+$": { + "type": "string" + }, + "^[a-z]+$": { + "type": "number" + } + } + } + }] + }`) + + out := NormalizeCodexToolSchemas(input) + tool := gjson.GetBytes(out, "tools.0") + params := tool.Get("parameters") + + // Key with \p{L}+ must be removed + patternProps := params.Get("patternProperties").Map() + if _, exists := patternProps[`^\p{L}+$`]; exists { + t.Errorf("expected patternProperties key '^\\\\p{L}+$' to be removed, got: %s", params.Get("patternProperties").Raw) + } + // Safe key must be preserved + if _, exists := patternProps[`^[a-z]+$`]; !exists { + t.Errorf("expected patternProperties key '^[a-z]+$' to be preserved, got: %s", params.Get("patternProperties").Raw) + } +} diff --git a/internal/translator/codex/claude/codex_claude_request.go b/internal/translator/codex/claude/codex_claude_request.go index b9a8a297e..18ac12003 100644 --- a/internal/translator/codex/claude/codex_claude_request.go +++ b/internal/translator/codex/claude/codex_claude_request.go @@ -726,7 +726,21 @@ func stripDialectKeywordsFromSchema(v any) { delete(schema, "pattern") } + // Inspect regex keys under patternProperties + if patternProps, ok := schema["patternProperties"].(map[string]any); ok { + for patternKey, subSchema := range patternProps { + if util.HasUnsupportedUnicodePropertyEscape(patternKey) { + delete(patternProps, patternKey) + } else { + stripDialectKeywordsFromSchema(subSchema) + } + } + } + for _, mapKey := range codexSchemaMapKeywords { + if mapKey == "patternProperties" { + continue + } if subMap, ok := schema[mapKey].(map[string]any); ok { for _, subSchema := range subMap { stripDialectKeywordsFromSchema(subSchema) diff --git a/internal/translator/codex/claude/codex_claude_request_test.go b/internal/translator/codex/claude/codex_claude_request_test.go index bd7f52d43..c26d9553d 100644 --- a/internal/translator/codex/claude/codex_claude_request_test.go +++ b/internal/translator/codex/claude/codex_claude_request_test.go @@ -1226,3 +1226,39 @@ func TestConvertClaudeRequestToCodex_StripsUnsupportedUnicodePropertyEscapePatte t.Errorf("expected required.0 == 'field', got %q", got) } } + +func TestConvertClaudeRequestToCodex_StripsPatternPropertiesIncompatibleKeys(t *testing.T) { + inputJSON := `{ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{ + "name": "pattern_tool", + "input_schema": { + "type": "object", + "patternProperties": { + "^\\\\p{L}+$": { + "type": "string" + }, + "^[a-z]+$": { + "type": "number" + } + } + } + }] + }` + + translated := ConvertClaudeRequestToCodex("gpt-5.6", []byte(inputJSON), false) + tools := gjson.GetBytes(translated, "tools").Array() + if len(tools) == 0 { + t.Fatalf("expected tools in translated payload, got: %s", translated) + } + params := tools[0].Get("parameters") + + patternProps := params.Get("patternProperties").Map() + if _, exists := patternProps[`^\p{L}+$`]; exists { + t.Errorf("expected patternProperties key '^\\\\p{L}+$' to be removed, got: %s", params.Get("patternProperties").Raw) + } + if _, exists := patternProps[`^[a-z]+$`]; !exists { + t.Errorf("expected patternProperties key '^[a-z]+$' to be preserved, got: %s", params.Get("patternProperties").Raw) + } +} diff --git a/internal/translator/openai/claude/openai_claude_request.go b/internal/translator/openai/claude/openai_claude_request.go index b4ae3d7a4..5fcb9575f 100644 --- a/internal/translator/openai/claude/openai_claude_request.go +++ b/internal/translator/openai/claude/openai_claude_request.go @@ -392,7 +392,21 @@ func normalizeObjectSchemaProperties(schema any) any { delete(value, "pattern") } + // Inspect regex keys under patternProperties + if patternProps, ok := value["patternProperties"].(map[string]any); ok { + for patternKey, subSchema := range patternProps { + if util.HasUnsupportedUnicodePropertyEscape(patternKey) { + delete(patternProps, patternKey) + } else { + patternProps[patternKey] = normalizeObjectSchemaProperties(subSchema) + } + } + } + for _, mapKey := range util.SchemaMapKeywords { + if mapKey == "patternProperties" { + continue + } if subMap, ok := value[mapKey].(map[string]any); ok { for subKey, subSchema := range subMap { subMap[subKey] = normalizeObjectSchemaProperties(subSchema) diff --git a/internal/translator/openai/claude/openai_claude_request_test.go b/internal/translator/openai/claude/openai_claude_request_test.go index 9515229de..7f32df47c 100644 --- a/internal/translator/openai/claude/openai_claude_request_test.go +++ b/internal/translator/openai/claude/openai_claude_request_test.go @@ -1119,3 +1119,40 @@ func TestConvertClaudeRequestToOpenAI_PreservesNonSchemaPatternKeys(t *testing.T t.Errorf("expected enum.0.pattern preserved, got %q", got) } } + +func TestConvertClaudeRequestToOpenAI_StripsPatternPropertiesIncompatibleKeys(t *testing.T) { + inputJSON := []byte(`{ + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{ + "name": "pattern_tool", + "input_schema": { + "type": "object", + "patternProperties": { + "^\\\\p{L}+$": { + "type": "string" + }, + "^[a-z]+$": { + "type": "number" + } + } + } + }] + }`) + + output := ConvertClaudeRequestToOpenAI("gpt-5.6", inputJSON, false) + outputJSON := gjson.ParseBytes(output) + + params := outputJSON.Get("tools.0.function.parameters") + if !params.Exists() { + t.Fatalf("expected function.parameters in output: %s", output) + } + + patternProps := params.Get("patternProperties").Map() + if _, exists := patternProps[`^\p{L}+$`]; exists { + t.Errorf("expected patternProperties key '^\\\\p{L}+$' to be removed, got: %s", params.Get("patternProperties").Raw) + } + if _, exists := patternProps[`^[a-z]+$`]; !exists { + t.Errorf("expected patternProperties key '^[a-z]+$' to be preserved, got: %s", params.Get("patternProperties").Raw) + } +}