feat(hardening): P1 maintainability metrics baseline + freshness gate (#1561)

Epic P1. Adds maintainability-metrics script (typed-error adoption, createLogger coverage, hotpath console.error, files>400 LOC), merges maintainability block into hardening-inventory report, adds 30-day freshness gate to ci-parity-gate.sh, re-baselines burndown 2026-06-18. Local validate + validate:ci-parity green.
This commit is contained in:
Tam Nhu Tran committed 2026-06-18 18:48:12 -04:00
1 parent 27f4e89ad1
commit 95a2864ef3
7 files changed
+1290 -306

No files matched your search

+31
View File
@@ -57,6 +57,37 @@ if git show-ref --verify --quiet "refs/remotes/origin/$BASE_BRANCH"; then
fi
fi
# Hardening inventory freshness: the maintainability metrics artifact must be
# regenerated within 30 days so the burndown stays current. Runs only after the
# skip conditions above (CCS_SKIP_PREPUSH_GATE, detached HEAD, behind origin).
HARDENING_JSON="docs/reports/hardening-inventory.json"
if [[ ! -f "$HARDENING_JSON" ]]; then
echo "[X] Missing $HARDENING_JSON."
echo " Regenerate with: bun run report:hardening"
exit 1
fi
HARDENING_TS=""
# If the working-tree copy differs from HEAD (contributor regenerated but not
# yet committed), use filesystem mtime; otherwise use the last commit time,
# which is stable across CI clones (checkout resets mtimes) and so correctly
# flags a stale committed artifact.
if git diff --quiet -- "$HARDENING_JSON" 2>/dev/null && git diff --cached --quiet -- "$HARDENING_JSON" 2>/dev/null; then
HARDENING_TS=$(git log -1 --format=%ct -- "$HARDENING_JSON" 2>/dev/null)
else
HARDENING_TS=$(stat -f %m "$HARDENING_JSON" 2>/dev/null || stat -c %Y "$HARDENING_JSON" 2>/dev/null)
fi
if [[ -n "$HARDENING_TS" ]]; then
NOW_TS=$(date +%s)
AGE_DAYS=$(( (NOW_TS - HARDENING_TS) / 86400 ))
if (( AGE_DAYS > 30 )); then
echo "[X] Hardening inventory is stale (${AGE_DAYS}d old; max 30d)."
echo " Regenerate with: bun run report:hardening"
echo " Then commit docs/reports/hardening-inventory.{json,md}."
exit 1
fi
echo "[i] Hardening inventory fresh (${AGE_DAYS}d old; max 30d)."
fi
echo "[i] Running CI-parity local checks..."
# `set -euo pipefail` above makes every step fail fast. Keep these commands
# explicit so parity drift is visible when CI changes.
+62 -4
View File
@@ -3,6 +3,8 @@
const fs = require('fs');
const path = require('path');
const { collectMaintainabilityMetrics } = require('./maintainability-metrics.js');
const ROOT_DIR = path.resolve(__dirname, '..');
const SRC_DIR = path.join(ROOT_DIR, 'src');
const REPORT_DIR = path.join(ROOT_DIR, 'docs', 'reports');
@@ -430,6 +432,7 @@ function buildReport() {
.filter((file) => /shim|re-export|compat/i.test(path.basename(file)))
),
},
maintainability: collectMaintainabilityMetrics(ROOT_DIR),
};
}
@@ -489,6 +492,55 @@ function renderMarkdown(report) {
lines.push('- _none_');
}
const m = report.maintainability;
if (m) {
lines.push('## Maintainability Metrics');
lines.push('');
lines.push('| Metric | Value |');
lines.push('|---|---:|');
lines.push(
`| typed-error adoption (typed/total throws) | ${(m.typedErrors.adoptionRatio * 100).toFixed(1)}% (${m.typedErrors.typedThrows}/${m.typedErrors.totalThrows}) |`
);
lines.push(
`| typed-error adoption (P4 locked subdomains) | ${(m.typedErrorAdoption.ratio * 100).toFixed(1)}% (${m.typedErrorAdoption.numerator}/${m.typedErrorAdoption.denominator}), target 40% |`
);
lines.push(
`| hotpath console.error/warn occurrences | ${m.hotpathConsoleErrors.hotpathOccurrences} (${m.hotpathConsoleErrors.totalOccurrences} total, ${m.hotpathConsoleErrors.exemptOccurrences} CLI-UX exempt) |`
);
lines.push(`| hotpath console.error/warn files | ${m.hotpathConsoleErrors.filesAffected} |`);
lines.push(
`| files with createLogger | ${m.loggerCoverage.filesWithCreateLogger}/${m.loggerCoverage.totalSourceFiles} |`
);
lines.push(
`| subdomains with zero createLogger | ${m.loggerCoverage.subdomainsWithZeroCreateLogger.length} (${m.loggerCoverage.subdomainsWithZeroCreateLogger.join(', ') || 'none'}) |`
);
lines.push(`| files > 400 LOC | ${m.largeFiles.countOver400} |`);
lines.push(`| files > 600 LOC | ${m.largeFiles.countOver600} |`);
lines.push('');
lines.push('### Top Hotpath console.error/warn Files');
lines.push('');
lines.push('| File | console.error/warn |');
lines.push('|---|---:|');
for (const item of m.hotpathConsoleErrors.topFiles) {
lines.push(`| \`${item.file}\` | ${item.count} |`);
}
if (m.hotpathConsoleErrors.topFiles.length === 0) {
lines.push('| _none_ | 0 |');
}
lines.push('');
lines.push('### Files > 400 LOC (top 15)');
lines.push('');
lines.push('| File | LOC |');
lines.push('|---|---:|');
for (const item of m.largeFiles.topOver400) {
lines.push(`| \`${item.file}\` | ${item.loc} |`);
}
if (m.largeFiles.topOver400.length === 0) {
lines.push('| _none_ | 0 |');
}
lines.push('');
}
lines.push('');
return lines.join('\n');
}
@@ -510,17 +562,23 @@ function main() {
console.log(
`[hardening-inventory] legacy markers total=${report.legacyShim.totalMarkers}, files=${report.legacyShim.filesAffected}`
);
if (report.maintainability) {
const mt = report.maintainability;
console.log(
`[hardening-inventory] maintainability: typed-adoption=${(mt.typedErrors.adoptionRatio * 100).toFixed(1)}% (${mt.typedErrors.typedThrows}/${mt.typedErrors.totalThrows}), locked=${(mt.typedErrorAdoption.ratio * 100).toFixed(1)}% (${mt.typedErrorAdoption.numerator}/${mt.typedErrorAdoption.denominator}), console.error=${mt.hotpathConsoleErrors.hotpathOccurrences}, zero-logger-subdomains=${mt.loggerCoverage.subdomainsWithZeroCreateLogger.length}, >400LOC=${mt.largeFiles.countOver400}`
);
}
console.log(`[hardening-inventory] wrote ${relJson}`);
console.log(`[hardening-inventory] wrote ${relMd}`);
}
if (require.main === module) {
main();
}
module.exports = {
buildReport,
collectSyncCallSites,
renderMarkdown,
stripComments,
};
if (require.main === module) {
main();
}
+349
View File
@@ -0,0 +1,349 @@
#!/usr/bin/env node
/**
* Maintainability metrics collector for the CCS CLI maintainability epic.
*
* Computes the metrics the epic tracks across phases:
* - typed-error adoption (throw new <TypedError> vs throw new Error vs other)
* - createLogger coverage by subdomain (which subdomains have zero)
* - hotpath console.error / console.warn call-site count (CLI-UX exempt)
* - files > 400 / 600 LOC
* - typed-error adoption in the P4 LOCKED denominator subdomains
*
* Accuracy relies on stripping comments/strings before regex matching. We
* reuse hardening-inventory.js#stripComments for that. The require is lazy
* (inside sanitize()) because hardening-inventory.js requires THIS module at
* its top level; a top-level require back would capture hardening-inventory's
* partial module.exports during the load cycle (it reassigns module.exports at
* the bottom). A function-scope require resolves against the fully-loaded
* module at call time, so the cycle is harmless.
*
* Approximate by design (grep-based). Method documented in
* docs/hardening-debt-burndown.md. Re-baseline when the schema changes.
*/
const fs = require('fs');
const path = require('path');
const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']);
// CCSError subclasses (src/errors/error-types.ts). A `throw new <OneOfThese>`
// counts as TYPED. `throw new Error(...)` is plain. Any other `throw new X(`
// is "other" (error subclass outside the canonical taxonomy).
const TYPED_ERROR_CLASSES = new Set([
'CCSError',
'ConfigError',
'NetworkError',
'AuthError',
'BinaryError',
'ProviderError',
'ProfileError',
'ProxyError',
'MigrationError',
'UserAbortError',
'ValidationError',
'RetryableError',
]);
// P4 LOCKED denominator: typed-error adoption is measured ONLY over these
// subdomains so the >40% goal cannot be gamed by narrowing scope. Emits both
// numerator and denominator counts alongside the ratio.
const TYPED_ADOPTION_SUBDOMAINS = ['cliproxy/quota', 'cliproxy/auth', 'web-server/routes', 'auth'];
// CLI-UX print surfaces exempt from the hotpath console.error sweep (P3).
// Diagnostics here are legitimate user-facing terminal output, not loggable
// errors, and stay on stdout/stderr via utils/ui.
const CLI_UX_EXEMPT_PREFIXES = ['src/commands/', 'src/management/', 'src/utils/ui/'];
const THROW_NEW_REGEX = /\bthrow\s+new\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
const CONSOLE_ERR_WARN_REGEX = /\bconsole\s*\.\s*(?:error|warn)\s*\(/g;
const CREATE_LOGGER_REGEX = /\bcreateLogger\s*\(/;
function sanitize(sourceText) {
// Lazy require; see module header for the circular-dependency rationale.
return require('./hardening-inventory.js').stripComments(sourceText);
}
function toPosixPath(filePath) {
return filePath.split(path.sep).join('/');
}
function isSourceFile(filePath) {
return SOURCE_EXTENSIONS.has(path.extname(filePath));
}
function isTestFile(relPath) {
return (
/(?:^|\/)(?:__tests__|tests?)\//.test(relPath) ||
/\.test\./.test(relPath) ||
/\.spec\./.test(relPath)
);
}
function isCliUxExempt(relPath) {
return CLI_UX_EXEMPT_PREFIXES.some(function (prefix) {
return relPath.startsWith(prefix);
});
}
/**
* Subdomain key for a src-relative path.
* src/cliproxy/quota/x.ts -> "cliproxy/quota"
* src/auth/x.ts -> "auth"
* src/x.ts -> "<root>"
*/
function subdomainOf(relPath) {
const rest = relPath.replace(/^src\//, '');
const parts = rest.split('/');
if (parts[0] === 'cliproxy' && parts.length > 2) {
return parts[0] + '/' + parts[1];
}
return parts[0] || '<root>';
}
function round4(n) {
return Math.round(n * 10000) / 10000;
}
// Items may be file-shaped ({file,count}) or subdomain-shaped ({subdomain,count});
// the tiebreaker key is whichever label field is present.
function labelOf(item) {
return item.file || item.subdomain || '';
}
function topByCount(items, limit) {
return items
.slice()
.sort(function (a, b) {
return b.count - a.count || labelOf(a).localeCompare(labelOf(b));
})
.slice(0, limit);
}
/** Classify `throw new X(` occurrences in sanitized source. */
function classifyThrows(sourceText) {
const sanitized = sanitize(sourceText);
const counts = { typed: 0, plain: 0, other: 0, total: 0 };
const re = new RegExp(THROW_NEW_REGEX.source, 'g');
let match;
while ((match = re.exec(sanitized)) !== null) {
const identifier = match[1];
counts.total += 1;
if (identifier === 'Error') {
counts.plain += 1;
} else if (TYPED_ERROR_CLASSES.has(identifier)) {
counts.typed += 1;
} else {
counts.other += 1;
}
}
return counts;
}
/** Count console.error / console.warn call sites in sanitized source. */
function countConsoleErrors(sourceText) {
const sanitized = sanitize(sourceText);
const re = new RegExp(CONSOLE_ERR_WARN_REGEX.source, 'g');
return (sanitized.match(re) || []).length;
}
/** True if the file directly creates a logger via createLogger(...). */
function hasCreateLogger(sourceText) {
return CREATE_LOGGER_REGEX.test(sanitize(sourceText));
}
/** Raw line count of a source file (matches `wc -l` content semantics). */
function countLoc(sourceText) {
if (!sourceText) return 0;
const parts = sourceText.split(/\r?\n/);
return sourceText.endsWith('\n') ? parts.length - 1 : parts.length;
}
function walkFiles(dirPath) {
const output = [];
let entries;
try {
entries = fs.readdirSync(dirPath, { withFileTypes: true });
} catch (_err) {
return output;
}
for (const entry of entries) {
if (entry.name === 'node_modules' || entry.name === 'dist') continue;
const fullPath = path.join(dirPath, entry.name);
if (entry.isDirectory()) {
output.push.apply(output, walkFiles(fullPath));
continue;
}
if (entry.isFile() && isSourceFile(fullPath)) output.push(fullPath);
}
return output;
}
function emptyAggregates() {
return {
typedBySubdomain: {},
loggerBySubdomain: {},
hotpathByFile: [],
hotpathTotal: 0,
hotpathExempt: 0,
hotpathNonExempt: 0,
hotpathFilesAffected: 0,
filesWithLogger: 0,
totalSourceFiles: 0,
typedTotal: 0,
typedTyped: 0,
typedPlain: 0,
typedOther: 0,
over400: [],
over600: [],
};
}
function ensureBucket(obj, key, factory) {
if (!obj[key]) obj[key] = factory();
return obj[key];
}
/**
* Walk <rootDir>/src and aggregate maintainability metrics.
* Returns the `maintainability` block merged into the hardening inventory.
*/
function collectMaintainabilityMetrics(rootDir) {
const srcDir = path.join(rootDir, 'src');
const files = walkFiles(srcDir);
const agg = emptyAggregates();
for (const fullPath of files) {
const relPath = toPosixPath(path.relative(rootDir, fullPath));
if (isTestFile(relPath)) continue;
const sourceText = fs.readFileSync(fullPath, 'utf8');
const sub = subdomainOf(relPath);
const throws = classifyThrows(sourceText);
agg.typedTotal += throws.total;
agg.typedTyped += throws.typed;
agg.typedPlain += throws.plain;
agg.typedOther += throws.other;
if (throws.total > 0) {
const bucket = ensureBucket(agg.typedBySubdomain, sub, function () {
return { typed: 0, plain: 0, other: 0, total: 0 };
});
bucket.typed += throws.typed;
bucket.plain += throws.plain;
bucket.other += throws.other;
bucket.total += throws.total;
}
agg.totalSourceFiles += 1;
const hasLogger = hasCreateLogger(sourceText);
if (hasLogger) agg.filesWithLogger += 1;
const lc = ensureBucket(agg.loggerBySubdomain, sub, function () {
return { files: 0, withLogger: 0 };
});
lc.files += 1;
if (hasLogger) lc.withLogger += 1;
const ce = countConsoleErrors(sourceText);
if (ce > 0) {
agg.hotpathTotal += ce;
if (isCliUxExempt(relPath)) {
agg.hotpathExempt += ce;
} else {
agg.hotpathNonExempt += ce;
agg.hotpathFilesAffected += 1;
agg.hotpathByFile.push({ file: relPath, count: ce });
}
}
const loc = countLoc(sourceText);
if (loc > 600) agg.over600.push({ file: relPath, loc: loc });
if (loc > 400) agg.over400.push({ file: relPath, loc: loc });
}
const adoptionRatio = agg.typedTotal > 0 ? agg.typedTyped / agg.typedTotal : 0;
let numerator = 0;
let denominator = 0;
for (const sub of TYPED_ADOPTION_SUBDOMAINS) {
const bucket = agg.typedBySubdomain[sub];
if (bucket) {
numerator += bucket.typed;
denominator += bucket.total;
}
}
const typedAdoptionRatio = denominator > 0 ? numerator / denominator : 0;
const subdomainsWithZeroCreateLogger = Object.keys(agg.loggerBySubdomain)
.filter(function (k) {
return agg.loggerBySubdomain[k].files > 0 && agg.loggerBySubdomain[k].withLogger === 0;
})
.sort();
const topOver400 = agg.over400
.slice()
.sort(function (a, b) {
return b.loc - a.loc || a.file.localeCompare(b.file);
})
.slice(0, 15);
return {
typedErrors: {
totalThrows: agg.typedTotal,
typedThrows: agg.typedTyped,
plainThrows: agg.typedPlain,
otherThrows: agg.typedOther,
adoptionRatio: round4(adoptionRatio),
topSubdomainsByThrows: topByCount(
Object.keys(agg.typedBySubdomain).map(function (k) {
const v = agg.typedBySubdomain[k];
return { subdomain: k, count: v.total, typed: v.typed, plain: v.plain };
}),
10
),
},
typedErrorAdoption: {
subdomains: TYPED_ADOPTION_SUBDOMAINS.slice(),
numerator: numerator,
denominator: denominator,
ratio: round4(typedAdoptionRatio),
targetRatio: 0.4,
},
loggerCoverage: {
filesWithCreateLogger: agg.filesWithLogger,
totalSourceFiles: agg.totalSourceFiles,
coverageRatio: round4(
agg.totalSourceFiles > 0 ? agg.filesWithLogger / agg.totalSourceFiles : 0
),
subdomainsWithZeroCreateLogger: subdomainsWithZeroCreateLogger,
topSubdomainsByFiles: topByCount(
Object.keys(agg.loggerBySubdomain).map(function (k) {
const v = agg.loggerBySubdomain[k];
return { subdomain: k, count: v.files, withLogger: v.withLogger };
}),
10
),
},
hotpathConsoleErrors: {
totalOccurrences: agg.hotpathTotal,
exemptOccurrences: agg.hotpathExempt,
hotpathOccurrences: agg.hotpathNonExempt,
filesAffected: agg.hotpathFilesAffected,
topFiles: topByCount(agg.hotpathByFile, 15),
},
largeFiles: {
countOver400: agg.over400.length,
countOver600: agg.over600.length,
topOver400: topOver400,
},
};
}
module.exports = {
collectMaintainabilityMetrics: collectMaintainabilityMetrics,
classifyThrows: classifyThrows,
countConsoleErrors: countConsoleErrors,
hasCreateLogger: hasCreateLogger,
countLoc: countLoc,
TYPED_ERROR_CLASSES: TYPED_ERROR_CLASSES,
TYPED_ADOPTION_SUBDOMAINS: TYPED_ADOPTION_SUBDOMAINS,
};