diff --git a/.codebase-memory/.gitattributes b/.codebase-memory/.gitattributes new file mode 100644 index 0000000000..a4a0dbfa7d --- /dev/null +++ b/.codebase-memory/.gitattributes @@ -0,0 +1,3 @@ +# Auto-generated by codebase-memory-mcp +# Treat the compressed snapshot as binary; regenerate it to resolve conflicts. +graph.db.zst binary diff --git a/.codebase-memory/graph.db.zst b/.codebase-memory/graph.db.zst new file mode 100644 index 0000000000..4cf0c25de9 Binary files /dev/null and b/.codebase-memory/graph.db.zst differ diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0f76650ea2..e4712ddb6a 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -214,6 +214,12 @@ owned evidence and a green portable smoke must not be read as proof of them. ## Deep tier (nightly) +`codebase-graph-refresh.yml` runs at 02:30 UTC, regenerates the committed +`.codebase-memory/graph.db.zst` bootstrap snapshot from the default branch with +a checksum-pinned `codebase-memory-mcp`, and opens or updates a normal review +PR through the repository GitHub App. It never pushes generated state directly +to `main`; required checks and human approval remain in the path. + `nightly-deep-ci.yml` (04:00 UTC) reuses `platform-and-compat.yml`, `reborn-tests.yml`, and `reborn-e2e.yml` via `workflow_call` at full scope. `reborn-e2e.yml` owns the deterministic Reborn surface coverage used by pull @@ -271,7 +277,8 @@ the configuration: ### Nightly alerting One path only: `nightly-watchdog.yml` (08:00 UTC) checks the latest scheduled -run of each nightly — Nightly Deep CI, Reborn Playwright, IronClaw Stress. A +run of each nightly — Codebase Graph Refresh, Nightly Deep CI, Reborn +Playwright, IronClaw Stress. A run that is missing, stale (>26h: the cron didn't fire), or concluded anything but success posts a failure line (workflow, conclusion, failed job names, run link) to the Slack channel behind diff --git a/.github/workflows/codebase-graph-refresh.yml b/.github/workflows/codebase-graph-refresh.yml new file mode 100644 index 0000000000..018eb0c603 --- /dev/null +++ b/.github/workflows/codebase-graph-refresh.yml @@ -0,0 +1,176 @@ +name: Codebase Graph Refresh + +# Regenerate the shared codebase-memory bootstrap snapshot from the default +# branch and open a normal review PR. The GitHub App token is intentional: +# PRs created with GITHUB_TOKEN do not trigger the repository's required checks. + +on: + schedule: + - cron: "30 2 * * *" # Daily at 02:30 UTC, before the deep-CI batch. + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: codebase-graph-refresh + cancel-in-progress: false + +jobs: + refresh: + if: github.repository == 'nearai/ironclaw' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: write + pull-requests: write + steps: + - name: Mint GitHub App token + id: app + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }} + private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }} + + - name: Check out the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + token: ${{ steps.app.outputs.token }} + persist-credentials: true + + - name: Install pinned codebase-memory-mcp + env: + CBM_VERSION: v0.9.0 + CBM_ARCHIVE: codebase-memory-mcp-linux-amd64-portable.tar.gz + CBM_SHA256: 8459d5c9d1457f2c82de3de307ffc7641ecbba2dde893427be1e62eca8ef9b25 + run: | + set -euo pipefail + install_dir="${RUNNER_TEMP}/codebase-memory-mcp-bin" + archive="${RUNNER_TEMP}/${CBM_ARCHIVE}" + mkdir -p "${install_dir}" + curl --proto '=https' --tlsv1.2 -fsSLo "${archive}" \ + "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/${CBM_ARCHIVE}" + echo "${CBM_SHA256} ${archive}" | sha256sum --check --strict + tar -xzf "${archive}" -C "${install_dir}" codebase-memory-mcp + chmod 0755 "${install_dir}/codebase-memory-mcp" + echo "${install_dir}" >> "${GITHUB_PATH}" + "${install_dir}/codebase-memory-mcp" --version + + - name: Regenerate and validate graph snapshot + env: + CBM_CACHE_DIR: ${{ runner.temp }}/codebase-memory-cache + run: | + set -euo pipefail + codebase-memory-mcp cli index_repository \ + --repo-path "${GITHUB_WORKSPACE}" \ + --mode full \ + --persistence true + + python3 - <<'PY' + import json + import pathlib + import subprocess + + root = pathlib.Path.cwd() + artifact = json.loads( + (root / ".codebase-memory" / "artifact.json").read_text(encoding="utf-8") + ) + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + if artifact.get("commit") != head: + raise SystemExit( + "graph metadata commit does not match the checked-out default branch" + ) + if artifact.get("nodes", 0) <= 0 or artifact.get("edges", 0) <= 0: + raise SystemExit("graph metadata reports an empty graph") + PY + + test -s .codebase-memory/graph.db.zst + zstd --test .codebase-memory/graph.db.zst + git diff --check + + changed="$(git diff --name-only)" + if [ -n "${changed}" ] && [ "${changed}" != ".codebase-memory/graph.db.zst" ]; then + echo "Graph refresh changed unexpected tracked paths:" >&2 + printf '%s\n' "${changed}" >&2 + exit 1 + fi + + - name: Open graph refresh PR + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8 + with: + token: ${{ steps.app.outputs.token }} + add-paths: .codebase-memory/graph.db.zst + branch: automation/codebase-graph-refresh + delete-branch: true + commit-message: "chore(agents): refresh codebase knowledge graph" + title: "chore(agents): refresh codebase knowledge graph" + body: | + ## Summary + + - Refresh the committed codebase-memory bootstrap snapshot from the current default branch. + - Generated by the nightly `Codebase Graph Refresh` workflow; review and merge normally. + + ## Change Type + + - [x] CI/Infrastructure + + ## Linked Issue + + None. + + ## Validation + + - [x] Relevant tests pass: full graph indexing completed and reported non-zero nodes and edges. + - [x] Manual testing: snapshot metadata matches the indexed default-branch commit and `zstd --test` passes. + + ## Test Strategy + + User behavior: Agents cloning the repository receive a recent bootstrap graph. + + Risk areas: + - [x] Persistence + - [x] Side effect + + Tests added or updated: + - Unit or contract: Not applicable: generated binary snapshot only. + - Reborn integration: Not applicable: no Reborn behavior changes. + - Recorded fixture: Not applicable: no model behavior changes. + - Browser E2E: Not applicable: no browser behavior changes. + - Backend or runtime: Not applicable: no Ironclaw runtime behavior changes. + - Live canary: Not applicable: the generator is local and deterministic. + + What the tests prove: The snapshot is non-empty, readable, and indexed from the expected commit. + + Commands run: `codebase-memory-mcp cli index_repository --mode full --persistence true`; `zstd --test .codebase-memory/graph.db.zst`; `git diff --check`. + + ## Security Impact + + The generating workflow keeps credentials host-side and limits the committed path to `.codebase-memory/graph.db.zst`. + + ## Reborn Trust-Boundary Checklist + + N/A: agent bootstrap data only; no Reborn trust boundary changes. + + ## Database Impact + + None. + + ## Blast Radius + + Agent code-discovery bootstrap data only. + + ## Rollback Plan + + Close this PR, or revert the snapshot commit after merge. + + ## Review Follow-Through + + Confirm the snapshot size and CI results before merging. + + --- + + **Review track**: A (generated chore) diff --git a/.github/workflows/nightly-watchdog.yml b/.github/workflows/nightly-watchdog.yml index 67a2e2d297..5c7f055c1b 100644 --- a/.github/workflows/nightly-watchdog.yml +++ b/.github/workflows/nightly-watchdog.yml @@ -1,7 +1,7 @@ name: Nightly Watchdog -# Single alerting path for the scheduled nightlies (Nightly Deep CI, Reborn -# Playwright, IronClaw Stress): checks each workflow's latest scheduled run +# Single alerting path for the scheduled nightlies (Codebase Graph Refresh, +# Nightly Deep CI, Reborn Playwright, IronClaw Stress): checks each workflow's latest scheduled run # and posts failures to Slack. Watching from OUTSIDE the runs # is deliberate — an in-run alert job dies with its own run on a # startup_failure (zero jobs execute; Nightly Deep CI was dark from its @@ -32,6 +32,7 @@ jobs: fail-fast: false matrix: workflow: + - Codebase Graph Refresh - Nightly Deep CI - Reborn Playwright - IronClaw Stress diff --git a/.gitignore b/.gitignore index f0a733f2c1..b50643bd75 100644 --- a/.gitignore +++ b/.gitignore @@ -83,10 +83,12 @@ tests/fixtures/llm_traces/live/*.log .anvil/ .codegraph/ -# codebase-memory knowledge graph — build artifact, not source. -# Rebuilt from code via the codebase-memory MCP; per-environment, never committed. +# codebase-memory knowledge graph. Share only the compressed bootstrap snapshot; +# local metadata and runtime state remain per-environment. # See CLAUDE.md -> "Code Discovery". -.codebase-memory/ +.codebase-memory/* +!.codebase-memory/.gitattributes +!.codebase-memory/graph.db.zst # test-tools fixture-bundle build artifacts — regenerate with # scripts/build-test-tools.sh; only sources/manifests/schemas/prompts are tracked. diff --git a/CLAUDE.md b/CLAUDE.md index 8040d7b186..0fbf11d337 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,11 +6,11 @@ This repo can be indexed into a **codebase knowledge graph** (the `codebase-memory` MCP server) over `crates/`. For any *where-is / who-calls / how-does-data-flow / what-does-this-touch* question, **probe the graph before reaching for `Grep`** — text search cannot see cross-crate call chains. A WebUI feature normally crosses `webui → ProductSurface → product → composition → runtime`, with the frontend inside `ironclaw_webui`. -**Where it lives:** `.codebase-memory/graph.db.zst` — a **git-ignored build artifact, not source**. One per environment, rebuilt from code. Never commit it. +**Where it lives:** `.codebase-memory/graph.db.zst` is a committed, compressed bootstrap snapshot shared by the team. The MCP imports it and incrementally catches up to the current checkout; local databases and `.codebase-memory/artifact.json` remain git-ignored per-environment state. Refresh the shared snapshot from a clean `main` checkout with `index_repository(repo_path=".", persistence=true)`. **Freshness (check at the start of a discovery task):** run `bash scripts/codebase-graph.sh status` — it compares the graph's indexed commit against `HEAD`. Then: -- **Missing** → `index_repository(repo_path=".")` once to build it. -- **Stale** → `detect_changes(since="")` for the changed symbols + blast radius, or re-run `index_repository` to fully refresh. +- **Missing** → `index_repository(repo_path=".", persistence=true)` once to build it. +- **Stale** → `detect_changes(since="")` for the changed symbols + blast radius, or re-run `index_repository(repo_path=".", persistence=true)` to fully refresh the shared snapshot. - The graph is a point-in-time index — verify anything it asserts against live code before acting. **Discovery recipes (use these instead of `Grep` for code structure):** diff --git a/scripts/ci/reborn_pr_test_plan.py b/scripts/ci/reborn_pr_test_plan.py index 4de6268a6b..b812e85f67 100644 --- a/scripts/ci/reborn_pr_test_plan.py +++ b/scripts/ci/reborn_pr_test_plan.py @@ -51,7 +51,14 @@ FULL_EVENTS = {"merge_group", "push", "workflow_call", "workflow_dispatch", "sch # the only satisfiable behaviour for that class was "never edit it", which is # not a policy anyone chose. Classifying it is the fix; loosening the # fail-closed arm is not. -IGNORED_PREFIXES = ("docs/", ".claude/", ".github/ISSUE_TEMPLATE/") +IGNORED_PREFIXES = ( + "docs/", + ".claude/", + ".github/ISSUE_TEMPLATE/", + # Agent bootstrap data only. The compressed graph and its attributes do + # not change an Ironclaw crate, test, or runtime surface. + ".codebase-memory/", +) IGNORED_GUIDANCE_PATHS = { "tests/CLAUDE.md", "tests/integration/CLAUDE.md", @@ -168,6 +175,8 @@ PR_STATIC_CONTROL_PATHS = { # * `run-reborn-webui.sh` is a local developer launcher for the WebUI dev # server. It is referenced by no workflow at all (a search over # `.github/` finds nothing), so no lane can be selected for it. + # * `codebase-graph.sh` inspects agent-only graph metadata. It does not + # execute or select a Reborn product test surface. # * `build-wasm-extensions.sh` is named in the same # `has_direct_wasm_abi_risk` classifier, which both scopes and runs it, # and additionally has a Code Style self-test @@ -178,6 +187,7 @@ PR_STATIC_CONTROL_PATHS = { "scripts/build-wasm-extensions.sh", "scripts/check-version-bumps.sh", "scripts/run-reborn-webui.sh", + "scripts/codebase-graph.sh", # Container build inputs. `platform-and-compat.yml` keys `has_docker_risk` # off exactly this pair and owns the image build; Code Style additionally # proves every `include_str!` target is inside each build context diff --git a/scripts/ci/test_reborn_pr_test_plan.py b/scripts/ci/test_reborn_pr_test_plan.py index caf8c0430e..1b8fed81ed 100644 --- a/scripts/ci/test_reborn_pr_test_plan.py +++ b/scripts/ci/test_reborn_pr_test_plan.py @@ -707,6 +707,25 @@ class RebornPrTestPlanTests(unittest.TestCase): self.assertEqual(plan["root_partitions"], [], path) self.assertEqual(plan["integration_lanes"], [], path) + def test_codebase_memory_artifacts_select_no_rust_lane(self) -> None: + """Shared agent graph data has no Reborn product or test surface.""" + for path in ( + ".codebase-memory/.gitattributes", + ".codebase-memory/graph.db.zst", + ): + with self.subTest(path=path): + plan = self.plan("pull_request", [path]) + self.assertEqual(plan["mode"], "none", path) + self.assertEqual(plan["crate_buckets"], [], path) + self.assertEqual(plan["root_partitions"], [], path) + self.assertEqual(plan["integration_lanes"], [], path) + + paired = self.plan( + "pull_request", [path, "crates/alpha/src/lib.rs"] + ) + self.assertEqual(paired["mode"], "selected", path) + self.assertNotEqual(paired["crate_buckets"], [], path) + def test_repo_wide_test_guidance_selects_no_rust_lane(self) -> None: for path in ("tests/CLAUDE.md", "tests/integration/CLAUDE.md"): with self.subTest(path=path): @@ -773,12 +792,15 @@ class RebornPrTestPlanTests(unittest.TestCase): "scripts/reborn-e2e-rust.sh", "scripts/check-version-bumps.sh", "scripts/run-reborn-webui.sh", + "scripts/codebase-graph.sh", ".gitignore", ): with self.subTest(path=path): plan = self.plan("pull_request", [path]) self.assertEqual(plan["mode"], "none", path) self.assertEqual(plan["crate_buckets"], [], path) + self.assertEqual(plan["root_partitions"], [], path) + self.assertEqual(plan["integration_lanes"], [], path) # The plan must say *why*, so a future reader sees the # decision rather than a silent "nothing to run". self.assertTrue( diff --git a/scripts/codebase-graph.sh b/scripts/codebase-graph.sh index 66a93050f1..65e21d0795 100755 --- a/scripts/codebase-graph.sh +++ b/scripts/codebase-graph.sh @@ -2,10 +2,10 @@ # # codebase-graph.sh — freshness status for the codebase-memory knowledge graph. # -# The graph (.codebase-memory/graph.db.zst) is a git-ignored build artifact indexed -# by the codebase-memory MCP server. This script only INSPECTS freshness; the actual -# (re)indexing is done through the MCP tools invoked by an agent: -# - build: index_repository(repo_path=".") +# The graph (.codebase-memory/graph.db.zst) is a committed bootstrap snapshot indexed +# by the codebase-memory MCP server. Local artifact metadata remains git-ignored. This +# script only INSPECTS freshness; the actual (re)indexing is done through MCP tools: +# - build/refresh: index_repository(repo_path=".", persistence=true) # - delta/impact: detect_changes(since="") # # Usage: bash scripts/codebase-graph.sh [status] @@ -20,7 +20,7 @@ read_field() { python3 -c "import json,sys;print(json.load(open('$artifact')).ge status() { if [ ! -f "$artifact" ]; then echo "graph: MISSING (no .codebase-memory/artifact.json)" - echo "action: build it once — call index_repository(repo_path=\".\") via the codebase-memory MCP" + echo "action: import/build it — call index_repository(repo_path=\".\", persistence=true) via the codebase-memory MCP" return 2 fi @@ -44,12 +44,12 @@ status() { n="$(git rev-list --count "$indexed"..HEAD 2>/dev/null || echo '?')" echo "status: STALE — ${n} commit(s) behind HEAD" echo "action: delta — detect_changes(since=\"$indexed\") (changed symbols + blast radius)" - echo " or full refresh — index_repository(repo_path=\".\")" + echo " or full refresh — index_repository(repo_path=\".\", persistence=true)" return 1 fi echo "status: DIVERGED — indexed commit is not in current history (rebase/force-push?)" - echo "action: full re-index — index_repository(repo_path=\".\")" + echo "action: full re-index — index_repository(repo_path=\".\", persistence=true)" return 1 }