mirror of
https://github.com/Hmbown/DeepSeek-TUI.git
synced 2026-09-03 06:50:13 +08:00
67adf84697
Measure the provider-free model-facing contract through production-owned seams. Keep deterministic minimal prompts by visible mode, build each Plan/Act/Operate tool surface through the canonical turn registry and request planner with inert wiring, and record full and active catalog counts, bytes, token estimates, sorted names, and stable identities. Add a hermetic representative prompt fixture that cumulatively introduces project authority, configured instructions, one workspace skill, memory, goal, and handoff. Normalize temporary roots before hashing every stage, require stable repeated identities, prove each prior marker appears exactly once while future markers remain absent, and ratchet adjacent stage deltas plus the final envelope. Enforce 55 one-way numeric ceilings and exact structural identities, while retaining the observable repeated skill-discovery baseline and future cached-zero payload. Reject equal-size tool substitutions, removals, missing identity, cross-mode swaps, and representative-stage substitutions; keep atomic permission-preserving budget updates and the locked offline heavy-lint gate.
51 lines
1.3 KiB
Python
Executable File
51 lines
1.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Measure canonical production-built tool catalogs by visible mode.
|
|
|
|
This delegates to an ignored Rust test that runs the production registry,
|
|
catalog, and active-request planner with inert wiring. Token counts are
|
|
deterministic estimates using ceil(serialized_bytes/4).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
MARKER = "TOOL_CATALOG_METRICS "
|
|
|
|
|
|
def main() -> int:
|
|
cmd = [
|
|
"cargo",
|
|
"test",
|
|
"--locked",
|
|
"-p",
|
|
"codewhale-tui",
|
|
"--bin",
|
|
"codewhale-tui",
|
|
"print_mode_tool_catalog_metrics",
|
|
"--",
|
|
"--ignored",
|
|
"--nocapture",
|
|
"--test-threads=1",
|
|
]
|
|
proc = subprocess.run(cmd, text=True, capture_output=True, check=False)
|
|
sys.stderr.write(proc.stderr)
|
|
|
|
combined = proc.stdout.splitlines() + proc.stderr.splitlines()
|
|
for line in combined:
|
|
if MARKER in line:
|
|
metrics = json.loads(line.split(MARKER, 1)[1])
|
|
print(json.dumps(metrics, indent=2, sort_keys=True))
|
|
return proc.returncode
|
|
|
|
sys.stdout.write(proc.stdout)
|
|
sys.stderr.write("missing TOOL_CATALOG_METRICS marker\n")
|
|
return proc.returncode or 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|