Merge and publish coverage reports (microsoft#6217)

* Upload coverage results * Merge and upload to azdo * News entry

Author: DonJayamanne

build/.nycrc

@@ -1,7 +1,6 @@
 {
  "extends": "@istanbuljs/nyc-config-typescript",
  "all": true,
- "reporter": ["lcov", "html", "text", "text-summary", "cobertura"],
  "include": [
  "src/client/**/*.ts", "src/test/**/*.js",
  "src/datascience-ui/**/*.ts", "src/datascience-ui/**/*.js"

build/ci/templates/generate_upload_coverage.yml

@@ -0,0 +1,17 @@
+steps:
+ # Generate the coverage reports.
+ - bash: |
+ npm run test:cover:report
+ displayName: 'run test:cover:report'
+ condition: contains(variables['TestsToRun'], 'testUnitTests')
+ failOnStderr: false
+
+
+ # Publish Code Coverage Results
+ - task: PublishCodeCoverageResults@1
+ displayName: 'Publish test:unittests coverage results'
+ condition: contains(variables['TestsToRun'], 'testUnitTests')
+ inputs:
+ codeCoverageTool: 'cobertura'
+ summaryFileLocation: "$(System.DefaultWorkingDirectory)/coverage/cobertura-coverage.xml"
+ reportDirectory: "$(System.DefaultWorkingDirectory)/coverage"

build/ci/templates/merge_upload_coverage.yml

@@ -0,0 +1,51 @@
+steps:
+ - bash: |
+ printenv
+ displayName: "Show all env vars"
+ condition: eq(variables['system.debug'], 'true')
+
+ - task: NodeTool@0
+ displayName: "Use Node $(NodeVersion)"
+ inputs:
+ versionSpec: $(NodeVersion)
+
+ - task: Npm@1
+ displayName: "Use NPM $(NpmVersion)"
+ inputs:
+ command: custom
+ verbose: true
+ customCommand: "install -g npm@$(NpmVersion)"
+
+ - task: Npm@1
+ displayName: "npm ci"
+ inputs:
+ command: custom
+ verbose: true
+ customCommand: ci
+
+ - task: DownloadBuildArtifacts@0
+ inputs:
+ buildType: "current"
+ allowPartiallySucceededBuilds: true
+ downloadType: "Specific"
+ itemPattern: "**/.nyc_output/**"
+ downloadPath: "$(Build.SourcesDirectory)"
+ displayName: "Restore Coverage Info"
+ condition: always()
+
+ # Now that we have downloaded artificats from `coverage-output-`, copy them
+ # into the root directory (they'll go under `.nyc_output/...`)
+ # These are the coverage output files that can be merged and then we can generate a report from them.
+ # This step results in downloading all individual `./nyc_output` results in coverage
+ # from all different test outputs. 
+ # Running the process of generating reports, will result in generation of 
+ # reports from all coverage data, i.e. we're basically combining all to generate a single merged report.
+ - bash: |
+ cp -r coverage-output-*/ ./
+ cd ./.nyc_output
+ ls -dlU .*/ */
+ ls
+ displayName: "Copy ./.nyc_output"
+ condition: always()
+
+ - template: generate_upload_coverage.yml

build/ci/templates/test_phases.yml

@@ -151,14 +151,21 @@ steps:
  buildPlatform: '$(Agent.Os)-Py$(pythonVersion)'
  buildConfiguration: 'UnitTests'
 
- # Publish Code Coverage Results
- - task: PublishCodeCoverageResults@1
- displayName: 'Publish test:unittests coverage results'
+ - task: CopyFiles@2
+ inputs:
+ sourceFolder: "$(Build.SourcesDirectory)/.nyc_output"
+ targetFolder: "$(Build.ArtifactStagingDirectory)/nyc/.nyc_output"
+ displayName: "Copy nyc_output to publish as artificat"
  condition: contains(variables['TestsToRun'], 'testUnitTests')
+
+ # Upload Code Coverage Results (to be merged later).
+ - task: PublishBuildArtifacts@1
  inputs:
- codeCoverageTool: 'cobertura'
- summaryFileLocation: "$(System.DefaultWorkingDirectory)/coverage/cobertura-coverage.xml"
- reportDirectory: "$(System.DefaultWorkingDirectory)/coverage"
+ pathtoPublish: "$(Build.ArtifactStagingDirectory)/nyc"
+ artifactName: 'coverage-output-$(Agent.Os)'
+ condition: contains(variables['TestsToRun'], 'testUnitTests')
+
+ - template: generate_upload_coverage.yml
 
  # Install the requirements for the Python or the system tests. This includes the supporting libs that
  # we ship in our extension such as PTVSD and Jedi.

build/ci/vscode-python-ci.yaml

@@ -310,3 +310,13 @@ jobs:
 
  steps:
  - template: templates/test_phases.yml
+
+- job: Coverage
+ dependsOn:
+ - CI
+ pool:
+ vmImage: "macos-latest"
+ variables:
+ TestsToRun: 'testUnitTests'
+ steps:
+ - template: templates/merge_upload_coverage.yml

build/ci/vscode-python-nightly-ci.yaml

@@ -476,3 +476,13 @@ jobs:
 
  steps:
  - template: templates/test_phases.yml
+
+- job: Coverage
+ dependsOn:
+ - Nightly
+ pool:
+ vmImage: "macos-latest"
+ variables:
+ TestsToRun: 'testUnitTests'
+ steps:
+ - template: templates/merge_upload_coverage.yml

build/ci/vscode-python-pr-validation.yaml

@@ -115,3 +115,13 @@ jobs:
 
  steps:
  - template: templates/test_phases.yml
+
+- job: Coverage
+ dependsOn:
+ - PR
+ pool:
+ vmImage: "macos-latest"
+ variables:
+ TestsToRun: 'testUnitTests'
+ steps:
+ - template: templates/merge_upload_coverage.yml

news/3 Code Health/6000.md

@@ -0,0 +1 @@
+Merge multiple coverage reports into one.

package.json

@@ -2178,9 +2178,10 @@
  "postinstall": "node ./node_modules/vscode/bin/install && node ./build/ci/postInstall.js",
  "test": "node ./out/test/standardTest.js && node ./out/test/multiRootTest.js",
  "test:unittests": "mocha --opts ./build/.mocha.unittests.js.opts",
- "test:unittests:cover": "nyc --nycrc-path build/.nycrc mocha --opts ./build/.mocha.unittests.ts.opts",
+ "test:unittests:cover": "nyc --silent --no-clean --nycrc-path build/.nycrc mocha --opts ./build/.mocha.unittests.ts.opts",
  "test:functional": "mocha --require source-map-support/register --opts ./build/.mocha.functional.opts",
  "test:functional:cover": "npm run test:functional",
+ "test:cover:report": "nyc --nycrc-path build/.nycrc report --reporter=lcov --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura",
  "testDebugger": "node ./out/test/testBootstrap.js ./out/test/debuggerTest.js",
  "testSingleWorkspace": "node ./out/test/testBootstrap.js ./out/test/standardTest.js",
  "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js",