zed/script/randomized-test-minimize

133 lines
3.5 KiB
Text
Raw Normal View History

#!/usr/bin/env node --redirect-warnings=/dev/null
const fs = require('fs')
const path = require('path')
const {spawnSync} = require('child_process')
const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
const CARGO_TEST_ARGS = [
'--release',
'--lib',
'--package', 'collab',
'random_collaboration',
]
if (require.main === module) {
if (process.argv.length < 4) {
process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
process.exit(1)
}
minimizeTestPlan(
process.argv[2],
process.argv[3],
parseInt(process.argv[4]) || 0
);
}
function minimizeTestPlan(
inputPlanPath,
outputPlanPath,
startIndex = 0
) {
const tempPlanPath = inputPlanPath + '.try'
fs.copyFileSync(inputPlanPath, outputPlanPath)
let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
process.stderr.write("minimizing failing test plan...\n")
for (let ix = startIndex; ix < testPlan.length; ix++) {
// Skip 'MutateClients' entries, since they themselves are not single operations.
if (testPlan[ix].MutateClients) {
continue
}
// Remove a row from the test plan
const newTestPlan = testPlan.slice()
newTestPlan.splice(ix, 1)
fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
const failingSeed = runTests({
SEED: '0',
LOAD_PLAN: tempPlanPath,
SAVE_PLAN: tempPlanPath,
ITERATIONS: '500'
})
// If the test failed, keep the test plan with the removed row. Reload the test
// plan from the JSON file, since the test itself will remove any operations
// which are no longer valid before saving the test plan.
if (failingSeed != null) {
process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
fs.copyFileSync(tempPlanPath, outputPlanPath)
testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
ix--
} else {
process.stderr.write(` - keep.\n`)
}
}
fs.unlinkSync(tempPlanPath)
// Re-run the final minimized plan to get the correct failing seed.
// This is a workaround for the fact that the execution order can
// slightly change when replaying a test plan after it has been
// saved and loaded.
const failingSeed = runTests({
SEED: '0',
ITERATIONS: '5000',
LOAD_PLAN: outputPlanPath,
})
process.stderr.write(`final test plan: ${outputPlanPath}\n`)
process.stderr.write(`final seed: ${failingSeed}\n`)
return failingSeed
}
function buildTests() {
const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
stdio: 'inherit',
encoding: 'utf8',
env: {
...process.env,
}
});
if (status !== 0) {
throw new Error('build failed')
}
}
function runTests(env) {
const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
stdio: 'pipe',
encoding: 'utf8',
env: {
...process.env,
...env,
}
});
if (status !== 0) {
FAILING_SEED_REGEX.lastIndex = 0
const match = FAILING_SEED_REGEX.exec(stdout)
if (!match) {
process.stderr.write("test failed, but no failing seed found:\n")
process.stderr.write(stdout)
process.stderr.write('\n')
process.exit(1)
}
return match[1]
} else {
return null
}
}
function serializeTestPlan(plan) {
return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
}
exports.buildTests = buildTests
exports.runTests = runTests
exports.minimizeTestPlan = minimizeTestPlan