Merge pull request microsoft#8774 from Microsoft/parallel-tests

run tests in parallel by equally dividing them between workers

Author: vladima

Jakefile.js

@@ -680,9 +680,9 @@ function cleanTestDirs() {
 }
 
 // used to pass data from jake command line directly to run.js
-function writeTestConfigFile(tests, light, testConfigFile) {
- console.log('Running test(s): ' + tests);
- var testConfigContents = JSON.stringify({ test: [tests], light: light });
+function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile) {
+ var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder });
+ console.log('Running tests with config: ' + testConfigContents);
  fs.writeFileSync('test.config', testConfigContents);
 }
 
@@ -692,7 +692,7 @@ function deleteTemporaryProjectOutput() {
  }
 }
 
-function runConsoleTests(defaultReporter, defaultSubsets) {
+function runConsoleTests(defaultReporter, runInParallel) {
  cleanTestDirs();
  var debug = process.env.debug || process.env.d;
  tests = process.env.test || process.env.tests || process.env.t;
@@ -701,9 +701,22 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
  if(fs.existsSync(testConfigFile)) {
  fs.unlinkSync(testConfigFile);
  }
+ var workerCount, taskConfigsFolder;
+ if (runInParallel) {
+ // generate name to store task configuration files
+ var prefix = os.tmpdir() + "/ts-tests";
+ var i = 1;
+ do {
+ taskConfigsFolder = prefix + i;
+ i++;
+ } while (fs.existsSync(taskConfigsFolder));
+ fs.mkdirSync(taskConfigsFolder);
+
+ workerCount = process.env.workerCount || os.cpus().length;
+ }
 
- if (tests || light) {
- writeTestConfigFile(tests, light, testConfigFile);
+ if (tests || light || taskConfigsFolder) {
+ writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile);
  }
 
  if (tests && tests.toLocaleLowerCase() === "rwc") {
@@ -717,61 +730,93 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
 
  // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
  // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
- var subsetRegexes;
- if(defaultSubsets.length === 0) {
- subsetRegexes = [tests];
+ if(!runInParallel) {
+ tests = tests ? ' -g "' + tests + '"' : '';
+ var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
+ console.log(cmd);
+ exec(cmd, function () {
+ runLinter();
+ finish();
+ }, function(e, status) {
+ finish(status);
+ });
+ 
  }
  else {
- var subsets = tests ? tests.split("|") : defaultSubsets;
- subsetRegexes = subsets.map(function (sub) { return "^" + sub + ".*$"; });
- subsetRegexes.push("^(?!" + subsets.join("|") + ").*$");
- }
- var counter = subsetRegexes.length;
- var errorStatus;
- subsetRegexes.forEach(function (subsetRegex, i) {
- tests = subsetRegex ? ' -g "' + subsetRegex + '"' : '';
- var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
+ // run task to load all tests and partition them between workers 
+ var cmd = "mocha " + " -R min " + colors + run;
  console.log(cmd);
- function finish(status) {
- counter--;
- // save first error status
- if (status !== undefined && errorStatus === undefined) {
- errorStatus = status;
- }
-
- deleteTemporaryProjectOutput();
- if (counter !== 0 || errorStatus === undefined) {
- // run linter when last worker is finished
- if (lintFlag && counter === 0) {
- var lint = jake.Task['lint'];
- lint.addListener('complete', function () {
+ exec(cmd, function() {
+ // read all configuration files and spawn a worker for every config
+ var configFiles = fs.readdirSync(taskConfigsFolder);
+ var counter = configFiles.length;
+ var firstErrorStatus;
+ // schedule work for chunks
+ configFiles.forEach(function (f) {
+ var configPath = path.join(taskConfigsFolder, f);
+ var workerCmd = "mocha" + " -t " + testTimeout + " -R " + reporter + " " + colors + " " + run + " --config='" + configPath + "'";
+ console.log(workerCmd);
+ exec(workerCmd, finishWorker, finishWorker) 
+ });
+ 
+ function finishWorker(e, errorStatus) {
+ counter--;
+ if (firstErrorStatus === undefined && errorStatus !== undefined) {
+ firstErrorStatus = errorStatus;
+ }
+ if (counter !== 0) {
+ complete();
+ }
+ else {
+ // last worker clean everything and runs linter in case if there were no errors
+ deleteTemporaryProjectOutput();
+ jake.rmRf(taskConfigsFolder);
+ if (firstErrorStatus === undefined) {
+ runLinter();
  complete();
- });
- lint.invoke();
+ }
+ else {
+ failWithStatus(firstErrorStatus);
+ }
  }
- complete();
- }
- else {
- fail("Process exited with code " + status);
  }
+ });
+ }
+ 
+ function failWithStatus(status) {
+ fail("Process exited with code " + status);
+ }
+ 
+ function finish(errorStatus) {
+ deleteTemporaryProjectOutput();
+ if (errorStatus !== undefined) {
+ failWithStatus(errorStatus);
  }
- exec(cmd, function () {
- finish();
- }, function(e, status) {
- finish(status);
+ else {
+ complete();
+ }
+ }
+ function runLinter() {
+ if (!lintFlag) {
+ return;
+ }
+ var lint = jake.Task['lint'];
+ lint.addListener('complete', function () {
+ complete();
  });
- });
+ lint.invoke();
+ }
 }
 
 var testTimeout = 20000;
 desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
 task("runtests-parallel", ["build-rules", "tests", builtLocalDirectory], function() {
- runConsoleTests('min', ['compiler', 'conformance', 'Projects', 'fourslash']);
+ runConsoleTests('min', /*runInParallel*/ true);
 }, {async: true});
 
 desc("Runs the tests using the built run.js file. Optional arguments are: t[ests]=regex r[eporter]=[list|spec|json|<more>] d[ebug]=true color[s]=false lint=true.");
 task("runtests", ["build-rules", "tests", builtLocalDirectory], function() {
- runConsoleTests('mocha-fivemat-progress-reporter', []);
+ runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false);
 }, {async: true});
 
 desc("Generates code coverage data via instanbul");

src/harness/compilerRunner.ts

@@ -12,7 +12,7 @@ const enum CompilerTestType {
 
 class CompilerBaselineRunner extends RunnerBase {
  private basePath = "tests/cases";
- private testSuiteName: string;
+ private testSuiteName: TestRunnerKind;
  private errors: boolean;
  private emit: boolean;
  private decl: boolean;
@@ -41,6 +41,14 @@ class CompilerBaselineRunner extends RunnerBase {
  this.basePath += "/" + this.testSuiteName;
  }
 
+ public kind() {
+ return this.testSuiteName;
+ }
+
+ public enumerateTestFiles() {
+ return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
+ }
+
  private makeUnitName(name: string, root: string) {
  return ts.isRootedDiskPath(name) ? name : ts.combinePaths(root, name);
  };
@@ -391,7 +399,7 @@ class CompilerBaselineRunner extends RunnerBase {
 
  // this will set up a series of describe/it blocks to run between the setup and cleanup phases
  if (this.tests.length === 0) {
- const testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
+ const testFiles = this.enumerateTestFiles();
  testFiles.forEach(fn => {
  fn = fn.replace(/\\/g, "/");
  this.checkTestCodeOutput(fn);

src/harness/fourslashRunner.ts

@@ -11,7 +11,7 @@ const enum FourSlashTestType {
 
 class FourSlashRunner extends RunnerBase {
  protected basePath: string;
- protected testSuiteName: string;
+ protected testSuiteName: TestRunnerKind;
 
  constructor(private testType: FourSlashTestType) {
  super();
@@ -35,9 +35,17 @@ class FourSlashRunner extends RunnerBase {
  }
  }
 
+ public enumerateTestFiles() {
+ return this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
+ }
+
+ public kind() {
+ return this.testSuiteName;
+ }
+
  public initializeTests() {
  if (this.tests.length === 0) {
- this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
+ this.tests = this.enumerateTestFiles();
  }
 
  describe(this.testSuiteName + " tests", () => {

src/harness/projectsRunner.ts

@@ -36,11 +36,19 @@ interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult {
 }
 
 class ProjectRunner extends RunnerBase {
+
+ public enumerateTestFiles() {
+ return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+ }
+
+ public kind(): TestRunnerKind {
+ return "project";
+ }
+
  public initializeTests() {
  if (this.tests.length === 0) {
- const testFiles = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+ const testFiles = this.enumerateTestFiles();
  testFiles.forEach(fn => {
- fn = fn.replace(/\\/g, "/");
  this.runProjectTestCase(fn);
  });
  }