Speed up targeted opencode tests
Reduce avoidable setup costs in slow opencode tests while preserving reviewed coverage and recording the benchmark evidence for follow-up test-suite work.
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
// Full-suite timing harness for the test-speed research in ../../perf/test-suite.md.
|
||||
// Use this for periodic sanity checks; use profile-test-files.ts for discovery.
|
||||
// Env: BENCH_WARMUPS=0 BENCH_RUNS=1 bun run bench:test
|
||||
const warmups = Number(Bun.env.BENCH_WARMUPS ?? 0)
|
||||
const runs = Number(Bun.env.BENCH_RUNS ?? 1)
|
||||
const timings: number[] = []
|
||||
|
||||
if (!Number.isInteger(warmups) || warmups < 0) {
|
||||
console.error("BENCH_WARMUPS must be a non-negative integer")
|
||||
process.exit(1)
|
||||
}
|
||||
if (!Number.isInteger(runs) || runs < 1) {
|
||||
console.error("BENCH_RUNS must be a positive integer")
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
for (const index of Array.from({ length: warmups + runs }, (_, index) => index)) {
|
||||
const measured = index >= warmups
|
||||
const label = measured ? `run ${index - warmups + 1}/${runs}` : `warmup ${index + 1}/${warmups}`
|
||||
const start = performance.now()
|
||||
console.log(`bench:test ${label}`)
|
||||
|
||||
const proc = Bun.spawn(["bun", "test", "--timeout", "30000"], {
|
||||
cwd: import.meta.dir + "/..",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
env: Bun.env,
|
||||
})
|
||||
|
||||
const exitCode = await proc.exited
|
||||
if (exitCode !== 0) {
|
||||
console.error(`bench:test failed during ${label} with exit code ${exitCode}`)
|
||||
process.exit(exitCode)
|
||||
}
|
||||
|
||||
const seconds = (performance.now() - start) / 1000
|
||||
console.log(`bench:test ${label} ${seconds.toFixed(3)}s`)
|
||||
if (measured) timings.push(seconds)
|
||||
}
|
||||
|
||||
const sorted = timings.toSorted((a, b) => a - b)
|
||||
const median = sorted[Math.floor(sorted.length / 2)]
|
||||
const mean = timings.reduce((sum, timing) => sum + timing, 0) / timings.length
|
||||
const best = sorted[0] ?? median
|
||||
const worst = sorted.at(-1) ?? median
|
||||
|
||||
console.log(
|
||||
`bench:test median=${median.toFixed(3)}s mean=${mean.toFixed(3)}s best=${best.toFixed(3)}s worst=${worst.toFixed(3)}s`,
|
||||
)
|
||||
console.log(`METRIC test_suite_seconds=${median.toFixed(3)}`)
|
||||
console.log(`METRIC test_suite_best_seconds=${best.toFixed(3)}`)
|
||||
console.log(`METRIC test_suite_worst_seconds=${worst.toFixed(3)}`)
|
||||
@@ -0,0 +1,42 @@
|
||||
// Per-file profiler for finding candidate test-speed work; see ../../perf/test-suite.md
|
||||
// for the benchmark notes, kept wins, and discarded experiments.
|
||||
// Example: TEST_PROFILE_GLOB='test/server/**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test
|
||||
const pattern = Bun.env.TEST_PROFILE_GLOB ?? "test/**/*.test.{ts,tsx}"
|
||||
const limit = Number(Bun.env.TEST_PROFILE_LIMIT ?? 0)
|
||||
const timeout = Bun.env.TEST_PROFILE_TIMEOUT ?? "30000"
|
||||
const files = Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: import.meta.dir + "/..", onlyFiles: true }))
|
||||
.then((files) => files.toSorted())
|
||||
.then((files) => (limit > 0 ? files.slice(0, limit) : files))
|
||||
|
||||
const results = []
|
||||
for (const file of await files) {
|
||||
const start = performance.now()
|
||||
const proc = Bun.spawn(["bun", "test", "--timeout", timeout, file], {
|
||||
cwd: import.meta.dir + "/..",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
env: Bun.env,
|
||||
})
|
||||
const [output, error, exitCode] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
])
|
||||
const seconds = (performance.now() - start) / 1000
|
||||
results.push({ file, seconds, exitCode })
|
||||
console.log(`${exitCode === 0 ? "PASS" : "FAIL"} ${seconds.toFixed(3)}s ${file}`)
|
||||
if (exitCode !== 0) console.log((output + error).trim())
|
||||
}
|
||||
|
||||
const sorted = results.toSorted((a, b) => b.seconds - a.seconds)
|
||||
console.log("\nSlowest test files:")
|
||||
for (const result of sorted.slice(0, Number(Bun.env.TEST_PROFILE_TOP ?? 20))) {
|
||||
console.log(`${result.seconds.toFixed(3)}s ${result.exitCode === 0 ? "PASS" : "FAIL"} ${result.file}`)
|
||||
}
|
||||
|
||||
if (sorted[0]) {
|
||||
console.log(`METRIC slowest_test_file_seconds=${sorted[0].seconds.toFixed(3)}`)
|
||||
console.log(`METRIC profiled_test_files=${results.length}`)
|
||||
}
|
||||
|
||||
if (results.some((result) => result.exitCode !== 0)) process.exit(1)
|
||||
Reference in New Issue
Block a user