Speed up targeted opencode tests

Reduce avoidable setup costs in slow opencode tests while preserving reviewed coverage and recording the benchmark evidence for follow-up test-suite work.
This commit is contained in:
Kit Langton
2026-05-18 16:18:29 +00:00
committed by GitHub
parent 0a945219a9
commit 896ad7b884
18 changed files with 737 additions and 575 deletions
@@ -0,0 +1,52 @@
// Full-suite timing harness for the test-speed research in ../../perf/test-suite.md.
// Use this for periodic sanity checks; use profile-test-files.ts for discovery.
// Env: BENCH_WARMUPS=0 BENCH_RUNS=1 bun run bench:test
const warmups = Number(Bun.env.BENCH_WARMUPS ?? 0)
const runs = Number(Bun.env.BENCH_RUNS ?? 1)
const timings: number[] = []
if (!Number.isInteger(warmups) || warmups < 0) {
console.error("BENCH_WARMUPS must be a non-negative integer")
process.exit(1)
}
if (!Number.isInteger(runs) || runs < 1) {
console.error("BENCH_RUNS must be a positive integer")
process.exit(1)
}
for (const index of Array.from({ length: warmups + runs }, (_, index) => index)) {
const measured = index >= warmups
const label = measured ? `run ${index - warmups + 1}/${runs}` : `warmup ${index + 1}/${warmups}`
const start = performance.now()
console.log(`bench:test ${label}`)
const proc = Bun.spawn(["bun", "test", "--timeout", "30000"], {
cwd: import.meta.dir + "/..",
stdout: "inherit",
stderr: "inherit",
env: Bun.env,
})
const exitCode = await proc.exited
if (exitCode !== 0) {
console.error(`bench:test failed during ${label} with exit code ${exitCode}`)
process.exit(exitCode)
}
const seconds = (performance.now() - start) / 1000
console.log(`bench:test ${label} ${seconds.toFixed(3)}s`)
if (measured) timings.push(seconds)
}
const sorted = timings.toSorted((a, b) => a - b)
const median = sorted[Math.floor(sorted.length / 2)]
const mean = timings.reduce((sum, timing) => sum + timing, 0) / timings.length
const best = sorted[0] ?? median
const worst = sorted.at(-1) ?? median
console.log(
`bench:test median=${median.toFixed(3)}s mean=${mean.toFixed(3)}s best=${best.toFixed(3)}s worst=${worst.toFixed(3)}s`,
)
console.log(`METRIC test_suite_seconds=${median.toFixed(3)}`)
console.log(`METRIC test_suite_best_seconds=${best.toFixed(3)}`)
console.log(`METRIC test_suite_worst_seconds=${worst.toFixed(3)}`)
@@ -0,0 +1,42 @@
// Per-file profiler for finding candidate test-speed work; see ../../perf/test-suite.md
// for the benchmark notes, kept wins, and discarded experiments.
// Example: TEST_PROFILE_GLOB='test/server/**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test
const pattern = Bun.env.TEST_PROFILE_GLOB ?? "test/**/*.test.{ts,tsx}"
const limit = Number(Bun.env.TEST_PROFILE_LIMIT ?? 0)
const timeout = Bun.env.TEST_PROFILE_TIMEOUT ?? "30000"
const files = Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: import.meta.dir + "/..", onlyFiles: true }))
.then((files) => files.toSorted())
.then((files) => (limit > 0 ? files.slice(0, limit) : files))
const results = []
for (const file of await files) {
const start = performance.now()
const proc = Bun.spawn(["bun", "test", "--timeout", timeout, file], {
cwd: import.meta.dir + "/..",
stdout: "pipe",
stderr: "pipe",
env: Bun.env,
})
const [output, error, exitCode] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
])
const seconds = (performance.now() - start) / 1000
results.push({ file, seconds, exitCode })
console.log(`${exitCode === 0 ? "PASS" : "FAIL"} ${seconds.toFixed(3)}s ${file}`)
if (exitCode !== 0) console.log((output + error).trim())
}
const sorted = results.toSorted((a, b) => b.seconds - a.seconds)
console.log("\nSlowest test files:")
for (const result of sorted.slice(0, Number(Bun.env.TEST_PROFILE_TOP ?? 20))) {
console.log(`${result.seconds.toFixed(3)}s ${result.exitCode === 0 ? "PASS" : "FAIL"} ${result.file}`)
}
if (sorted[0]) {
console.log(`METRIC slowest_test_file_seconds=${sorted[0].seconds.toFixed(3)}`)
console.log(`METRIC profiled_test_files=${results.length}`)
}
if (results.some((result) => result.exitCode !== 0)) process.exit(1)