From 288dedd2a4ab4a7bed1ea2f663bf6463750fc7eb Mon Sep 17 00:00:00 2001 From: yuhao Date: Wed, 29 Apr 2026 12:00:19 +0000 Subject: [PATCH] Add automatic PR labeling --- .github/labeler.yml | 25 ++ .github/scripts/pr-labeler.js | 173 +++++++++++ .../scripts/tests/pr-labeler-fixtures.json | 270 ++++++++++++++++++ .github/scripts/tests/pr-labeler.test.js | 199 +++++++++++++ .github/workflows/pr-labeler-tests.yml | 43 +++ .github/workflows/pr-labeler.yml | 33 +++ .gitignore | 1 + 7 files changed, 744 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/scripts/pr-labeler.js create mode 100644 .github/scripts/tests/pr-labeler-fixtures.json create mode 100644 .github/scripts/tests/pr-labeler.test.js create mode 100644 .github/workflows/pr-labeler-tests.yml create mode 100644 .github/workflows/pr-labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..0cbe6f0ed --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,25 @@ +cli-anything-hub: + - changed-files: + - any-glob-to-any-file: + - "cli-hub/**" + - "docs/hub/**" + - "registry.json" + - "public_registry.json" + - "matrix_registry.json" + +cli-anything-skill: + - changed-files: + - any-glob-to-any-file: + - "cli-anything-plugin/**" + - "codex-skill/**" + - "cli-hub-meta-skill/**" + - "openclaw-skill/**" + - "skills/**" + - "**/SKILL.md" + +github-actions: + - changed-files: + - any-glob-to-any-file: + - ".github/workflows/**" + - ".github/labeler.yml" + - ".github/scripts/pr-labeler.js" diff --git a/.github/scripts/pr-labeler.js b/.github/scripts/pr-labeler.js new file mode 100644 index 000000000..25aa62950 --- /dev/null +++ b/.github/scripts/pr-labeler.js @@ -0,0 +1,173 @@ +const LABELS = { + "new-cli": { + color: "0E8A16", + description: "Adds a new CLI or generated harness", + }, + "existing-cli-fix": { + color: "1D76DB", + description: "Fixes or improves an existing CLI harness", + }, + "cli-anything-skill": { + color: "5319E7", + description: "Changes CLI-Anything plugin or skill files", + }, + "cli-anything-hub": { + color: "FBCA04", + description: "Changes CLI-Hub, registries, or hub docs", + }, + "documentation": { + color: "0075CA", + description: "Documentation issue or improvement", + }, + "github-actions": { + color: "6F42C1", + description: "Changes GitHub Actions or automation", + }, +}; + +const SCRIPT_MANAGED_LABELS = ["new-cli", "existing-cli-fix", "documentation"]; +const REGISTRY_FILES = new Set([ + "registry.json", + "public_registry.json", + "matrix_registry.json", +]); + +function isHarnessFile(path) { + return /^[^/]+\/agent-harness\//.test(path); +} + +function isNewHarnessManifest(file) { + return ( + file.status === "added" && + /^[^/]+\/agent-harness\/(setup\.py|pyproject\.toml)$/.test(file.filename) + ); +} + +function isDocumentationFile(path) { + if (/(^|\/)SKILL\.md$/i.test(path)) { + return false; + } + + return ( + /^README(?:_[A-Z]+)?\.md$/.test(path) || + /^(CONTRIBUTING|SECURITY)\.md$/.test(path) || + /^docs\//.test(path) || + /^[^/]+\.md$/i.test(path) + ); +} + +function titleLooksLikeRegistryCli(title) { + return /\b(add|introduce|new)\b/i.test(title) && /\b(cli|harness|registry)\b/i.test(title); +} + +function computeScriptLabels(files, title) { + const paths = files.map((file) => file.filename); + const labelsToApply = new Set(); + + const hasHarnessChange = paths.some(isHarnessFile); + const hasNewHarness = files.some(isNewHarnessManifest); + const registryOnly = paths.length > 0 && paths.every((path) => REGISTRY_FILES.has(path)); + const registryNewCli = registryOnly && titleLooksLikeRegistryCli(title || ""); + const documentationOnly = paths.length > 0 && paths.every(isDocumentationFile); + + if (hasNewHarness || registryNewCli) { + labelsToApply.add("new-cli"); + } else if (hasHarnessChange) { + labelsToApply.add("existing-cli-fix"); + } + + if (documentationOnly) { + labelsToApply.add("documentation"); + } + + return labelsToApply; +} + +async function ensureLabels(github, owner, repo, core) { + const existing = await github.paginate(github.rest.issues.listLabelsForRepo, { + owner, + repo, + per_page: 100, + }); + const existingNames = new Set(existing.map((label) => label.name)); + + for (const [name, definition] of Object.entries(LABELS)) { + if (existingNames.has(name)) { + continue; + } + + core.info(`Creating missing label: ${name}`); + await github.rest.issues.createLabel({ + owner, + repo, + name, + color: definition.color, + description: definition.description, + }); + } +} + +async function syncScriptLabels(github, owner, repo, pullNumber, currentLabels, labelsToApply, core) { + for (const label of labelsToApply) { + if (currentLabels.has(label)) { + continue; + } + + core.info(`Adding label: ${label}`); + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: pullNumber, + labels: [label], + }); + } + + for (const label of SCRIPT_MANAGED_LABELS) { + if (!currentLabels.has(label) || labelsToApply.has(label)) { + continue; + } + + core.info(`Removing label: ${label}`); + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number: pullNumber, + name: label, + }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + } +} + +module.exports = async ({ github, context, core }) => { + const pullRequest = context.payload.pull_request; + if (!pullRequest) { + core.info("No pull_request payload found; skipping PR labeling."); + return; + } + + const { owner, repo } = context.repo; + const pullNumber = pullRequest.number; + + await ensureLabels(github, owner, repo, core); + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: pullNumber, + per_page: 100, + }); + + const labelsToApply = computeScriptLabels(files, pullRequest.title); + + const currentLabels = new Set((pullRequest.labels || []).map((label) => label.name)); + await syncScriptLabels(github, owner, repo, pullNumber, currentLabels, labelsToApply, core); +}; + +module.exports.computeScriptLabels = computeScriptLabels; +module.exports.LABELS = LABELS; +module.exports.SCRIPT_MANAGED_LABELS = SCRIPT_MANAGED_LABELS; diff --git a/.github/scripts/tests/pr-labeler-fixtures.json b/.github/scripts/tests/pr-labeler-fixtures.json new file mode 100644 index 000000000..475624694 --- /dev/null +++ b/.github/scripts/tests/pr-labeler-fixtures.json @@ -0,0 +1,270 @@ +[ + { + "number": 262, + "title": "Add MseeP.ai badge", + "files": [ + {"filename": "README.md", "status": "modified"} + ], + "expected": ["documentation"] + }, + { + "number": 260, + "title": "feat: add hacker-feeds-cli to registry", + "files": [ + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "new-cli"] + }, + { + "number": 259, + "title": "feat: add ve-twini to registry", + "files": [ + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "new-cli"] + }, + { + "number": 258, + "title": "feat: add sliver to registry", + "files": [ + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "new-cli"] + }, + { + "number": 256, + "title": "feat(Calibre): add a CLI-Anything harness for Calibre Desktop", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "README.md", "status": "modified"}, + {"filename": "calibre/agent-harness/CALIBRE.md", "status": "added"}, + {"filename": "calibre/agent-harness/cli_anything/calibre/calibre_cli.py", "status": "added"}, + {"filename": "calibre/agent-harness/cli_anything/calibre/core/session.py", "status": "added"}, + {"filename": "calibre/agent-harness/cli_anything/calibre/skills/SKILL.md", "status": "added"}, + {"filename": "calibre/agent-harness/setup.py", "status": "added"}, + {"filename": "cli-anything-plugin/scripts/setup-cli-anything.sh", "status": "modified"}, + {"filename": "codex-skill/scripts/install.sh", "status": "modified"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "skills/cli-anything-calibre/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 254, + "title": "feat: add quietshrink harness - Apple Silicon screen recording compressor", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "quietshrink/agent-harness/QUIETSHRINK.md", "status": "added"}, + {"filename": "quietshrink/agent-harness/cli_anything/quietshrink/quietshrink_cli.py", "status": "added"}, + {"filename": "quietshrink/agent-harness/cli_anything/quietshrink/skills/SKILL.md", "status": "added"}, + {"filename": "quietshrink/agent-harness/setup.py", "status": "added"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "skills/cli-anything-quietshrink/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 252, + "title": "Add cli-anything-rekordbox: Pioneer Rekordbox 6/7 harness", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "ableton/agent-harness/setup.py", "status": "added"}, + {"filename": "ableton/agent-harness/cli_anything/ableton/ableton_cli.py", "status": "added"}, + {"filename": "rekordbox/agent-harness/setup.py", "status": "added"}, + {"filename": "rekordbox/agent-harness/cli_anything/rekordbox/rekordbox_cli.py", "status": "added"}, + {"filename": "rekordbox/agent-harness/cli_anything/rekordbox/skills/SKILL.md", "status": "added"}, + {"filename": "serum/agent-harness/setup.py", "status": "added"}, + {"filename": "vital/agent-harness/setup.py", "status": "added"}, + {"filename": "skills/cli-anything-rekordbox/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-skill", "new-cli"] + }, + { + "number": 251, + "title": "feat: add s&box CLI harness", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "README.md", "status": "modified"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "sbox/agent-harness/SBOX.md", "status": "added"}, + {"filename": "sbox/agent-harness/cli_anything/sbox/sbox_cli.py", "status": "added"}, + {"filename": "sbox/agent-harness/cli_anything/sbox/skills/SKILL.md", "status": "added"}, + {"filename": "sbox/agent-harness/setup.py", "status": "added"}, + {"filename": "skills/cli-anything-sbox/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 245, + "title": "feat: mature lldb agent harness", + "files": [ + {"filename": "lldb/agent-harness/LLDB.md", "status": "modified"}, + {"filename": "lldb/agent-harness/cli_anything/lldb/lldb_cli.py", "status": "modified"}, + {"filename": "lldb/agent-harness/cli_anything/lldb/skills/SKILL.md", "status": "modified"}, + {"filename": "lldb/agent-harness/setup.py", "status": "modified"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "skills/cli-anything-lldb/SKILL.md", "status": "modified"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"] + }, + { + "number": 244, + "title": "feat: improve Unreal Insights live analysis", + "files": [ + {"filename": "skills/cli-anything-unrealinsights/SKILL.md", "status": "modified"}, + {"filename": "unrealinsights/agent-harness/UNREALINSIGHTS.md", "status": "modified"}, + {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/core/analyze.py", "status": "added"}, + {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/skills/SKILL.md", "status": "modified"}, + {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/unrealinsights_cli.py", "status": "modified"} + ], + "expected": ["cli-anything-skill", "existing-cli-fix"] + }, + { + "number": 241, + "title": "Feat/add firefly iii cli", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "firefly-iii/agent-harness/README.md", "status": "added"}, + {"filename": "firefly-iii/agent-harness/cli_anything/firefly_iii/firefly_iii_cli.py", "status": "added"}, + {"filename": "firefly-iii/agent-harness/cli_anything/firefly_iii/skills/SKILL.md", "status": "added"}, + {"filename": "firefly-iii/agent-harness/setup.py", "status": "added"}, + {"filename": "firefly-iii/agent-harness/skills/cli-anything-firefly-iii/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-skill", "new-cli"] + }, + { + "number": 240, + "title": "fix(blender): align docs and render execute with real behavior", + "files": [ + {"filename": "README.md", "status": "modified"}, + {"filename": "README_CN.md", "status": "modified"}, + {"filename": "blender/agent-harness/cli_anything/blender/README.md", "status": "modified"}, + {"filename": "blender/agent-harness/cli_anything/blender/blender_cli.py", "status": "modified"}, + {"filename": "blender/agent-harness/cli_anything/blender/core/render.py", "status": "modified"}, + {"filename": "blender/agent-harness/cli_anything/blender/utils/blender_backend.py", "status": "modified"} + ], + "expected": ["existing-cli-fix"] + }, + { + "number": 238, + "title": "feat: add NSLogger CLI harness", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "README.md", "status": "modified"}, + {"filename": "nslogger/agent-harness/NSLOGGER.md", "status": "added"}, + {"filename": "nslogger/agent-harness/cli_anything/nslogger/nslogger_cli.py", "status": "added"}, + {"filename": "nslogger/agent-harness/cli_anything/nslogger/skills/SKILL.md", "status": "added"}, + {"filename": "nslogger/agent-harness/setup.py", "status": "added"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "skills/cli-anything-nslogger/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 237, + "title": "refactor(openclaw-skill->macrocli): rename, complete backends, add recorder and visual anchor", + "files": [ + {"filename": "macrocli/SKILL.md", "status": "added"}, + {"filename": "macrocli/agent-harness/MACROCLI.md", "status": "added"}, + {"filename": "macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py", "status": "added"}, + {"filename": "macrocli/agent-harness/cli_anything/macrocli/skills/SKILL.md", "status": "added"}, + {"filename": "macrocli/agent-harness/setup.py", "status": "added"}, + {"filename": "openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py", "status": "modified"}, + {"filename": "registry.json", "status": "modified"}, + {"filename": "skills/cli-anything-macrocli/SKILL.md", "status": "added"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 233, + "title": "fix(browser): add MCP timeout guard to prevent fs hangs", + "files": [ + {"filename": "browser/agent-harness/cli_anything/browser/browser_cli.py", "status": "modified"}, + {"filename": "browser/agent-harness/cli_anything/browser/skills/SKILL.md", "status": "modified"}, + {"filename": "browser/agent-harness/cli_anything/browser/utils/domshell_backend.py", "status": "modified"}, + {"filename": "browser/agent-harness/setup.py", "status": "modified"}, + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"] + }, + { + "number": 220, + "title": "fix(browser): harden bridge and sync REPL template", + "files": [ + {"filename": ".github/workflows/pr-ci.yml", "status": "added"}, + {"filename": "browser/agent-harness/cli_anything/browser/browser_cli.py", "status": "modified"}, + {"filename": "browser/agent-harness/cli_anything/browser/utils/repl_skin.py", "status": "modified"}, + {"filename": "cli-anything-plugin/repl_skin.py", "status": "modified"}, + {"filename": "gimp/agent-harness/cli_anything/gimp/utils/repl_skin.py", "status": "modified"} + ], + "expected": ["cli-anything-skill", "existing-cli-fix", "github-actions"] + }, + { + "number": 218, + "title": "fix: Executing cli-anything-browser --json fs ls yields no return result", + "files": [ + {"filename": "browser/agent-harness/cli_anything/browser/tests/test_core.py", "status": "modified"}, + {"filename": "browser/agent-harness/cli_anything/browser/utils/domshell_backend.py", "status": "modified"} + ], + "expected": ["existing-cli-fix"] + }, + { + "number": 202, + "title": "security: add user confirmation guard to meta-skill installation flow", + "files": [ + {"filename": "cli-hub-meta-skill/SKILL.md", "status": "modified"}, + {"filename": "docs/hub/SKILL.md", "status": "modified"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill"] + }, + { + "number": 189, + "title": "feat: add MiniMax CLI harness (chat + TTS)", + "files": [ + {"filename": ".gitignore", "status": "modified"}, + {"filename": "minimax/agent-harness/cli_anything/minimax/minimax_cli.py", "status": "added"}, + {"filename": "minimax/agent-harness/cli_anything/minimax/skills/SKILL.md", "status": "added"}, + {"filename": "minimax/agent-harness/setup.py", "status": "added"}, + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"] + }, + { + "number": 187, + "title": "Refactor AnyGen CLI for better state management", + "files": [ + {"filename": "anygen/agent-harness/cli_anything/anygen/anygen_cli.py", "status": "modified"} + ], + "expected": ["existing-cli-fix"] + }, + { + "number": 167, + "title": "feat(freecad): add FreeCAD 1.0.2 backward compatibility", + "files": [ + {"filename": "freecad/agent-harness/FREECAD.md", "status": "modified"}, + {"filename": "freecad/agent-harness/cli_anything/freecad/core/generate.py", "status": "modified"}, + {"filename": "freecad/agent-harness/cli_anything/freecad/freecad_cli.py", "status": "modified"}, + {"filename": "freecad/agent-harness/cli_anything/freecad/skills/SKILL.md", "status": "modified"}, + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"] + }, + { + "number": 106, + "title": "fix: split token-aware into default exclusions + optional --economic mode", + "files": [ + {"filename": "cli-anything-plugin/HARNESS.md", "status": "modified"}, + {"filename": "cli-anything-plugin/commands/cli-anything.md", "status": "modified"}, + {"filename": "cli-anything-plugin/commands/refine.md", "status": "modified"} + ], + "expected": ["cli-anything-skill"] + }, + { + "number": "registry-maintenance", + "title": "Update registry dates", + "files": [ + {"filename": "registry.json", "status": "modified"} + ], + "expected": ["cli-anything-hub"] + } +] diff --git a/.github/scripts/tests/pr-labeler.test.js b/.github/scripts/tests/pr-labeler.test.js new file mode 100644 index 000000000..887e7d863 --- /dev/null +++ b/.github/scripts/tests/pr-labeler.test.js @@ -0,0 +1,199 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const test = require("node:test"); + +const labeler = require("../pr-labeler.js"); +const fixtures = require("./pr-labeler-fixtures.json"); + +const REPO_ROOT = path.resolve(__dirname, "../../.."); +const LABELER_CONFIG_PATH = path.join(REPO_ROOT, ".github/labeler.yml"); +const EXPECTED_LABELS = [ + "cli-anything-hub", + "cli-anything-skill", + "documentation", + "existing-cli-fix", + "github-actions", + "new-cli", +]; + +function normalizeLabels(labels) { + return [...labels].sort(); +} + +function sameLabels(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function loadLabelerPatterns() { + const patternsByLabel = new Map(); + let currentLabel = null; + + for (const line of fs.readFileSync(LABELER_CONFIG_PATH, "utf8").split(/\r?\n/)) { + const labelMatch = line.match(/^([a-z0-9-]+):$/); + if (labelMatch) { + currentLabel = labelMatch[1]; + patternsByLabel.set(currentLabel, []); + continue; + } + + const patternMatch = line.match(/-\s+"([^"]+)"/); + if (currentLabel && patternMatch) { + patternsByLabel.get(currentLabel).push(patternMatch[1]); + } + } + + return patternsByLabel; +} + +function globToRegex(pattern) { + let regex = "^"; + + for (let index = 0; index < pattern.length; index += 1) { + const char = pattern[index]; + const next = pattern[index + 1]; + const afterNext = pattern[index + 2]; + + if (char === "*" && next === "*" && afterNext === "/") { + regex += "(?:.*/)?"; + index += 2; + continue; + } + + if (char === "*" && next === "*") { + regex += ".*"; + index += 1; + continue; + } + + if (char === "*") { + regex += "[^/]*"; + continue; + } + + regex += char.replace(/[|\\{}()[\]^$+?.]/g, "\\$&"); + } + + regex += "$"; + return new RegExp(regex); +} + +function computePathLabels(files) { + const labels = new Set(); + const patternsByLabel = loadLabelerPatterns(); + + for (const [label, patterns] of patternsByLabel.entries()) { + const regexes = patterns.map(globToRegex); + if (files.some((file) => regexes.some((regex) => regex.test(file.filename)))) { + labels.add(label); + } + } + + return labels; +} + +function computeAllLabels(sample) { + const labels = new Set(labeler.computeScriptLabels(sample.files, sample.title)); + for (const pathLabel of computePathLabels(sample.files)) { + labels.add(pathLabel); + } + return normalizeLabels(labels); +} + +function summarizeMetrics(results) { + const labels = new Set(EXPECTED_LABELS); + for (const result of results) { + for (const label of result.expected) labels.add(label); + for (const label of result.predicted) labels.add(label); + } + + const perLabel = {}; + for (const label of labels) { + let truePositive = 0; + let falsePositive = 0; + let falseNegative = 0; + + for (const result of results) { + const expected = new Set(result.expected); + const predicted = new Set(result.predicted); + + if (expected.has(label) && predicted.has(label)) truePositive += 1; + if (!expected.has(label) && predicted.has(label)) falsePositive += 1; + if (expected.has(label) && !predicted.has(label)) falseNegative += 1; + } + + const precisionDenominator = truePositive + falsePositive; + const recallDenominator = truePositive + falseNegative; + + perLabel[label] = { + truePositive, + falsePositive, + falseNegative, + precision: precisionDenominator === 0 ? 1 : truePositive / precisionDenominator, + recall: recallDenominator === 0 ? 1 : truePositive / recallDenominator, + }; + } + + const exactMatches = results.filter((result) => sameLabels(result.predicted, result.expected)).length; + + return { + exactAccuracy: exactMatches / results.length, + perLabel, + }; +} + +test("real PR fixture label accuracy and recall", () => { + const results = fixtures.map((sample) => ({ + number: sample.number, + title: sample.title, + expected: normalizeLabels(sample.expected), + predicted: computeAllLabels(sample), + })); + + const mismatches = results.filter((result) => { + return !sameLabels(result.predicted, result.expected); + }); + + const metrics = summarizeMetrics(results); + + assert.deepStrictEqual(mismatches, [], JSON.stringify({mismatches, metrics}, null, 2)); + assert.equal(metrics.exactAccuracy, 1); + + for (const [label, metric] of Object.entries(metrics.perLabel)) { + assert.equal(metric.precision, 1, `${label} precision: ${JSON.stringify(metric)}`); + assert.equal(metric.recall, 1, `${label} recall: ${JSON.stringify(metric)}`); + } +}); + +test("labeler config and script agree on the supported label set", () => { + const configLabels = new Set(loadLabelerPatterns().keys()); + const scriptLabels = new Set(Object.keys(labeler.LABELS)); + for (const label of EXPECTED_LABELS) { + assert(scriptLabels.has(label), `${label} must be creatable by pr-labeler.js`); + } + + assert(configLabels.has("cli-anything-hub")); + assert(configLabels.has("cli-anything-skill")); + assert(configLabels.has("github-actions")); +}); + +test("registry-only maintenance is not treated as a new CLI", () => { + const labels = computeAllLabels({ + title: "Update registry dates", + files: [{filename: "registry.json", status: "modified"}], + }); + + assert.deepStrictEqual(labels, ["cli-anything-hub"]); +}); + +test("mixed README and harness changes are not documentation-only", () => { + const labels = computeAllLabels({ + title: "fix(blender): update docs and render behavior", + files: [ + {filename: "README.md", status: "modified"}, + {filename: "blender/agent-harness/cli_anything/blender/core/render.py", status: "modified"}, + ], + }); + + assert.deepStrictEqual(labels, ["existing-cli-fix"]); +}); diff --git a/.github/workflows/pr-labeler-tests.yml b/.github/workflows/pr-labeler-tests.yml new file mode 100644 index 000000000..c3e41f423 --- /dev/null +++ b/.github/workflows/pr-labeler-tests.yml @@ -0,0 +1,43 @@ +name: PR Labeler Tests + +on: + pull_request: + paths: + - ".github/labeler.yml" + - ".github/scripts/pr-labeler.js" + - ".github/scripts/tests/pr-labeler-fixtures.json" + - ".github/scripts/tests/pr-labeler.test.js" + - ".github/workflows/pr-labeler.yml" + - ".github/workflows/pr-labeler-tests.yml" + push: + branches: + - main + paths: + - ".github/labeler.yml" + - ".github/scripts/pr-labeler.js" + - ".github/scripts/tests/pr-labeler-fixtures.json" + - ".github/scripts/tests/pr-labeler.test.js" + - ".github/workflows/pr-labeler.yml" + - ".github/workflows/pr-labeler-tests.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Validate PR labeler script + run: node --check .github/scripts/pr-labeler.js + + - name: Run PR labeler tests + run: node .github/scripts/tests/pr-labeler.test.js diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml new file mode 100644 index 000000000..0214c63e8 --- /dev/null +++ b/.github/workflows/pr-labeler.yml @@ -0,0 +1,33 @@ +name: Pull Request Labeler + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + label: + runs-on: ubuntu-latest + steps: + - name: Check out base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + persist-credentials: false + + - name: Apply computed labels + uses: actions/github-script@v7 + with: + script: | + const labelPullRequest = require("./.github/scripts/pr-labeler.js"); + await labelPullRequest({ github, context, core }); + + - name: Apply path labels + uses: actions/labeler@v6 + with: + configuration-path: .github/labeler.yml + sync-labels: true diff --git a/.gitignore b/.gitignore index 5ea93cc6f..d87b6ce41 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ !/.github/workflows/ !/.github/scripts/ !/.github/CODEOWNERS +!/.github/labeler.yml !/.github/PULL_REQUEST_TEMPLATE.md !/.github/ISSUE_TEMPLATE/