Merge branch 'fixUnitTest'

* fixUnitTest: fixed issues in running unittests

Author: DonJayamanne

pythonFiles/PythonTools/visualstudio_py_testlauncher.py

@@ -203,6 +203,7 @@ def main():
  parser.add_option('-p', '--port', type='int', metavar='<port>', help='listen for debugger connections on <port>')
  parser.add_option('-x', '--mixed-mode', action='store_true', help='wait for mixed-mode debugger to attach')
  parser.add_option('-t', '--test', type='str', dest='tests', action='append', help='specifies a test to run')
+ parser.add_option('--testFile', type='str', help='Fully qualitified path to file name')
  parser.add_option('-c', '--coverage', type='str', help='enable code coverage and specify filename')
  parser.add_option('-r', '--result-port', type='int', help='connect to port on localhost and send test results')
  parser.add_option('--us', type='str', help='Directory to start discovery')
@@ -264,14 +265,48 @@ def main():
  cov.start()
  except:
  pass
- if opts.tests is None:
+ if opts.tests is None and opts.testFile is None:
  if opts.us is None:
  opts.us = '.'
  if opts.up is None:
  opts.up = 'test*.py'
  tests = unittest.defaultTestLoader.discover(opts.us, opts.up)
  else:
- tests = unittest.defaultTestLoader.loadTestsFromNames(opts.tests)
+ # loadTestsFromNames doesn't work well (with duplicate file names or class names) 
+ # Easier approach is find the test suite and use that for running
+ loader = unittest.TestLoader()
+ # opts.us will be passed in
+ suites = loader.discover(opts.us, pattern=os.path.basename(opts.testFile))
+ suite = None
+ tests = None 
+ if opts.tests is None:
+ # Run everything in the test file
+ tests = suites
+ else:
+ # Run a specific test class or test method
+ for suite in suites._tests:
+ for cls in suite._tests: 
+ try:
+ for m in cls._tests:
+ testId = m.id()
+ if testId.startswith(opts.tests[0]):
+ suite = cls
+ if testId == opts.tests[0]:
+ tests = m
+ break
+ except Exception as err:
+ errorMessage = traceback.format_exception() 
+ pass
+ if tests is None:
+ tests = suite
+ if tests is None and suite is None:
+ _channel.send_event(
+ name='error', 
+ outcome='',
+ traceback = '',
+ message = 'Failed to identify the test',
+ test = ''
+ )
  if opts.uvInt is None:
  opts.uvInt = 0
  runner = unittest.TextTestRunner(verbosity=opts.uvInt, resultclass=VsTestResult)

pythonFiles/preview/jedi

@@ -0,0 +1 @@
+Subproject commit b594a7d861af662fcb6487c9cada20dc7857e89b

src/client/unittests/nosetest/collector.ts

@@ -77,10 +77,9 @@ export function discoverTests(rootDirectory: string, args: string[], token: Canc
  });
  }
 
- return execPythonFile(pythonSettings.unitTest.nosetestPath, args.concat(['--collect-only', '-vvv']), rootDirectory, true, processOutput)
- .then(() => {
- // process the last entry
- parseNoseTestModuleCollectionResult(rootDirectory, logOutputLines, testFiles);
+ return execPythonFile(pythonSettings.unitTest.nosetestPath, args.concat(['--collect-only', '-vvv']), rootDirectory, true)
+ .then(data => {
+ processOutput(data);
  // Exclude tests that don't have any functions or test suites
  let indices = testFiles.filter(testFile => {
  return testFile.suites.length === 0 && testFile.functions.length === 0;

src/client/unittests/nosetest/runner.ts

@@ -1,6 +1,5 @@
 'use strict';
 import * as path from 'path';
-import {execPythonFile} from './../../common/utils';
 import {createDeferred, createTemporaryFile} from '../../common/helpers';
 import {OutputChannel, window, CancellationToken} from 'vscode';
 import {TestFile, TestsToRun, TestSuite, TestFunction, FlattenedTestFunction, Tests, TestStatus, FlattenedTestSuite} from '../common/contracts';

src/client/unittests/pytest/collector.ts

@@ -87,14 +87,12 @@ export function discoverTests(rootDirectory: string, args: string[], token: vsco
  });
  }
 
- return execPythonFile(pythonSettings.unitTest.pyTestPath, args.concat(['--collect-only']), rootDirectory, false, processOutput, token)
- .then(() => {
+ return execPythonFile(pythonSettings.unitTest.pyTestPath, args.concat(['--collect-only']), rootDirectory, false, null, token)
+ .then(data => {
+ processOutput(data);
  if (token && token.isCancellationRequested) {
  return Promise.reject<Tests>('cancelled');
  }
-
- // process the last entry
- parsePyTestModuleCollectionResult(rootDirectory, logOutputLines, testFiles, parentNodes);
  return flattenTestFiles(testFiles);
  });
 }

src/client/unittests/pytest/runner.ts

@@ -1,7 +1,6 @@
 /// <reference path="../../../../typings/globals/xml2js/index.d.ts" />
 
 'use strict';
-import {execPythonFile} from './../../common/utils';
 import {createDeferred, createTemporaryFile} from '../../common/helpers';
 import {TestFile, TestsToRun, TestSuite, TestFunction, FlattenedTestFunction, Tests, TestStatus, FlattenedTestSuite} from '../common/contracts';
 import {extractBetweenDelimiters, flattenTestFiles, updateResults, convertFileToPackage} from '../common/testUtils';

src/client/unittests/unittest/collector.ts

@@ -43,11 +43,14 @@ export function discoverTests(rootDirectory: string, args: string[], token: vsco
  const pythonScript = `import unittest
 loader = unittest.TestLoader()
 suites = loader.discover("${startDirectory}", pattern="${pattern}")
-print("start")
+print("start") #Don't remove this line
 for suite in suites._tests:
  for cls in suite._tests:
- for m in cls._tests:
- print(m.id())`;
+ try:
+ for m in cls._tests:
+ print(m.id())
+ except:
+ pass`;
 
  let startedCollecting = false;
  let testItems: string[] = [];
@@ -63,20 +66,25 @@ for suite in suites._tests:
  return;
  }
  line = line.trim();
- if (line.length === 0){
+ if (line.length === 0) {
  return;
  }
  testItems.push(line);
  });
  }
  args = [];
- return execPythonFile(pythonSettings.pythonPath, args.concat(['-c', pythonScript]), rootDirectory, true, processOutput, token)
- .then(() => {
+ return execPythonFile(pythonSettings.pythonPath, args.concat(['-c', pythonScript]), rootDirectory, true, null, token)
+ .then(data => {
+ processOutput(data);
  if (token && token.isCancellationRequested) {
  return Promise.reject<Tests>('cancelled');
  }
 
- return parseTestIds(rootDirectory, testItems);
+ let testsDirectory = rootDirectory;
+ if (startDirectory.length > 1) {
+ testsDirectory = path.isAbsolute(startDirectory) ? startDirectory : path.resolve(rootDirectory, startDirectory);
+ }
+ return parseTestIds(testsDirectory, testItems);
  });
 }
 
@@ -109,7 +117,7 @@ function addTestId(rootDirectory: string, testId: string, testFiles: TestFile[])
  fullPath: filePath,
  functions: [],
  suites: [],
- nameToRun: paths.join('.'),
+ nameToRun: `${className}.${functionName}`,
  xmlName: '',
  status: TestStatus.Idle,
  time: 0
@@ -118,7 +126,7 @@ function addTestId(rootDirectory: string, testId: string, testFiles: TestFile[])
  }
 
  // Check if we already have this test file
- const classNameToRun = testIdParts.slice(0, testIdParts.length - 1).join('.');
+ const classNameToRun = className;
  let testSuite = testFile.suites.find(cls => cls.nameToRun === classNameToRun);
  if (!testSuite) {
  testSuite = {

src/client/unittests/unittest/runner.ts

@@ -2,7 +2,6 @@
 
 'use strict';
 import * as path from 'path';
-import {execPythonFile} from './../../common/utils';
 import {createDeferred, createTemporaryFile} from '../../common/helpers';
 import {TestFile, TestsToRun, TestSuite, TestFunction, FlattenedTestFunction, Tests, TestStatus, FlattenedTestSuite} from '../common/contracts';
 import {extractBetweenDelimiters, flattenTestFiles, updateResults, convertFileToPackage} from '../common/testUtils';
@@ -66,18 +65,54 @@ export function runTest(rootDirectory: string, tests: Tests, args: string[], tes
  for (let counter = 0; counter < testPaths.length; counter++) {
  testPaths[counter] = '-t' + testPaths[counter].trim();
  }
- let testArgs = buildTestArgs(args);
- testArgs = [testLauncherFile].concat(testArgs).concat(`--result-port=${port}`).concat(testPaths);
- return run(settings.pythonPath, testArgs, rootDirectory, token, outChannel);
+ const startTestDiscoveryDirectory = getStartDirectory(args);
+
+ function runTest(testFile: string = '', testId: string = '') {
+ let testArgs = buildTestArgs(args);
+ testArgs.push(`--result-port=${port}`);
+ testArgs.push(`--us=${startTestDiscoveryDirectory}`);
+ if (testId.length > 0) {
+ testArgs.push(`-t${testId}`);
+ }
+ if (testFile.length > 0) {
+ testArgs.push(`--testFile=${testFile}`);
+ }
+ return run(settings.pythonPath, [testLauncherFile].concat(testArgs), rootDirectory, token, outChannel);
+ }
+
+ // Test everything
+ if (testPaths.length === 0) {
+ return runTest();
+ }
+
+ // Ok, the ptvs test runner can only work with one test at a time
+ let promise = Promise.resolve<string>('');
+ if (Array.isArray(testsToRun.testFile)) {
+ testsToRun.testFile.forEach(testFile => {
+ promise = promise.then(() => runTest(testFile.fullPath, testFile.nameToRun));
+ });
+ }
+ if (Array.isArray(testsToRun.testSuite)) {
+ testsToRun.testSuite.forEach(testSuite => {
+ const testFileName = tests.testSuits.find(t => t.testSuite === testSuite).parentTestFile.fullPath;
+ promise = promise.then(() => runTest(testFileName, testSuite.nameToRun));
+ });
+ }
+ if (Array.isArray(testsToRun.testFunction)) {
+ testsToRun.testFunction.forEach(testFn => {
+ const testFileName = tests.testFunctions.find(t => t.testFunction === testFn).parentTestFile.fullPath;
+ promise = promise.then(() => runTest(testFileName, testFn.nameToRun));
+ });
+ }
+ return promise;
  }).then(() => {
  updateResults(tests);
  return tests;
  });
 }
 
-function buildTestArgs(args: string[]): string[] {
+function getStartDirectory(args: string[]): string {
  let startDirectory = '.';
- let pattern = 'test*.py';
  const indexOfStartDir = args.findIndex(arg => arg.indexOf('-s') === 0 || arg.indexOf('--start-directory') === 0);
  if (indexOfStartDir >= 0) {
  const startDir = args[indexOfStartDir].trim();
@@ -93,6 +128,11 @@ function buildTestArgs(args: string[]): string[] {
  }
  }
  }
+ return startDirectory;
+}
+function buildTestArgs(args: string[]): string[] {
+ const startTestDiscoveryDirectory = getStartDirectory(args);
+ let pattern = 'test*.py';
  const indexOfPattern = args.findIndex(arg => arg.indexOf('-p') === 0 || arg.indexOf('--pattern') === 0);
  if (indexOfPattern >= 0) {
  const patternValue = args[indexOfPattern].trim();
@@ -110,7 +150,7 @@ function buildTestArgs(args: string[]): string[] {
  }
  const failFast = args.some(arg => arg.trim() === '-f' || arg.trim() === '--failfast');
  const verbosity = args.some(arg => arg.trim().indexOf('-v') === 0) ? 2 : 1;
- const testArgs = [`--us=${startDirectory}`, `--up=${pattern}`, `--uvInt=${verbosity}`];
+ const testArgs = [`--us=${startTestDiscoveryDirectory}`, `--up=${pattern}`, `--uvInt=${verbosity}`];
  if (failFast) {
  testArgs.push('--uf');
  }