zed/script/randomized-test-minimize

#!/usr/bin/env node --redirect-warnings=/dev/null

const fs = require('fs')
const path = require('path')
const {spawnSync} = require('child_process')

const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
const CARGO_TEST_ARGS = [
  '--release',
  '--lib',
  '--package', 'collab',
  'random_collaboration',
]

if (require.main === module) {
  if (process.argv.length < 4) {
    process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
    process.exit(1)
  }

  minimizeTestPlan(
    process.argv[2],
    process.argv[3],
    parseInt(process.argv[4]) || 0
  );
}

function minimizeTestPlan(
  inputPlanPath,
  outputPlanPath,
  startIndex = 0
) {
  const tempPlanPath = inputPlanPath + '.try'

  fs.copyFileSync(inputPlanPath, outputPlanPath)
  let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))

  process.stderr.write("minimizing failing test plan...\n")
  for (let ix = startIndex; ix < testPlan.length; ix++) {
    // Skip 'MutateClients' entries, since they themselves are not single operations.
    if (testPlan[ix].MutateClients) {
      continue
    }

    // Remove a row from the test plan
    const newTestPlan = testPlan.slice()
    newTestPlan.splice(ix, 1)
    fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');

    process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
    const failingSeed = runTests({
      SEED: '0',
      LOAD_PLAN: tempPlanPath,
      SAVE_PLAN: tempPlanPath,
      ITERATIONS: '500'
    })

    // If the test failed, keep the test plan with the removed row. Reload the test
    // plan from the JSON file, since the test itself will remove any operations
    // which are no longer valid before saving the test plan.
    if (failingSeed != null) {
      process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
      fs.copyFileSync(tempPlanPath, outputPlanPath)
      testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
      ix--
    } else {
      process.stderr.write(` - keep.\n`)
    }
  }

  fs.unlinkSync(tempPlanPath)

  // Re-run the final minimized plan to get the correct failing seed.
  // This is a workaround for the fact that the execution order can
  // slightly change when replaying a test plan after it has been
  // saved and loaded.
  const failingSeed = runTests({
    SEED: '0',
    ITERATIONS: '5000',
    LOAD_PLAN: outputPlanPath,
  })

  process.stderr.write(`final test plan: ${outputPlanPath}\n`)
  process.stderr.write(`final seed: ${failingSeed}\n`)
  return failingSeed
}

function buildTests() {
  const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
    stdio: 'inherit',
    encoding: 'utf8',
    env: {
      ...process.env,
    }
  });
  if (status !== 0) {
    throw new Error('build failed')
  }
}

function runTests(env) {
  const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
    stdio: 'pipe',
    encoding: 'utf8',
    env: {
      ...process.env,
      ...env,
    }
  });

  if (status !== 0) {
    FAILING_SEED_REGEX.lastIndex = 0
    const match = FAILING_SEED_REGEX.exec(stdout)
    if (!match) {
      process.stderr.write("test failed, but no failing seed found:\n")
      process.stderr.write(stdout)
      process.stderr.write('\n')
      process.exit(1)
    }
    return match[1]
  } else {
    return null
  }
}

function serializeTestPlan(plan) {
  return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
}

exports.buildTests = buildTests
exports.runTests = runTests
exports.minimizeTestPlan = minimizeTestPlan