chore: execute tests based on a dependency graph (#3653)

* chore: build project graph for determining which tests to run * chore: add graphviz visualization to it * chore: update project graph analysis and add affected test command * chore: run ava tests based on the files * chore: update github action with the new command and enhance tests * chore: fail with process.exit on catch * chore: update affected base sha to be the PRs merge into shasum * chore: change checkout depth * chore: update visiting plugins * chore: update error messaging * chore: remove snapshots as they dont work outside of tests dir * chore: change to array as format serializes sets differently on new node versions Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

Author: Lukas Holzer

.github/workflows/main.yml

@@ -35,6 +35,8 @@ jobs:
  if: "${{ matrix.os == 'windows-latest' }}"
  - name: Git checkout
  uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
  - name: Use Node.js ${{ matrix.node-version }}
  uses: actions/setup-node@v2
  with:
@@ -49,9 +51,18 @@ jobs:
  - name: Linting
  run: npm run format:ci
  if: "${{ matrix.node-version == '*' && !steps.release-check.outputs.IS_RELEASE}}"
+ - name: Determine Test Command
+ uses: haya14busa/action-cond@v1
+ id: testCommand
+ with:
+ cond: ${{ github.event_name == 'pull_request' }}
+ if_true: 'npm run test:affected ${{ github.event.pull_request.base.sha }}' # on pull requests test with the project graph only the affected tests
+ if_false: 'npm run test:ci' # on the base branch run all the tests as security measure
+ - name: Prepare tests
+ run: npm run test:init
  - name: Tests
  if: '${{ !steps.release-check.outputs.IS_RELEASE }}'
- run: npm run test:ci
+ run: ${{ steps.testCommand.outputs.value }}
  env:
  # GitHub secrets are not available when running on PR from forks
  # We set a flag so we can skip tests that access Netlify API

npm-shrinkwrap.json

@@ -56,14 +56,16 @@
  "format:check:prettier": "cross-env-shell prettier --check $npm_package_config_prettier",
  "format:fix:prettier": "cross-env-shell prettier --write $npm_package_config_prettier",
  "test:dev": "run-s test:init:* test:dev:*",
- "test:ci": "run-s test:init:* test:ci:*",
+ "test:ci": "run-s test:ci:*",
+ "test:init": "run-s test:init:*",
  "test:init:build": "run-s build:*",
  "test:init:cli-version": "npm run start -- --version",
  "test:init:cli-help": "npm run start -- --help",
  "test:init:eleventy-deps": "npm ci --prefix tests/eleventy-site --no-audit",
  "test:init:hugo-deps": "npm ci --prefix tests/hugo-site --no-audit",
  "test:dev:ava": "ava --verbose",
  "test:ci:ava": "nyc -r json ava",
+ "test:affected": "node ./tools/affected-test.js",
  "docs": "node ./site/scripts/docs.js",
  "watch": "nyc --reporter=lcov ava --watch",
  "build:manifest": "oclif-dev manifest",
@@ -202,12 +204,15 @@
  "@oclif/test": "^1.2.5",
  "ava": "^3.15.0",
  "eslint-plugin-sort-destructure-keys": "^1.3.5",
+ "fast-glob": "^3.2.7",
  "form-data": "^4.0.0",
  "from2-string": "^1.1.0",
  "got": "^11.8.1",
+ "graphviz": "^0.0.9",
  "ini": "^2.0.0",
  "jsonwebtoken": "^8.5.1",
  "mkdirp": "^1.0.4",
+ "mock-fs": "^5.1.2",
  "nyc": "^15.0.0",
  "p-timeout": "^4.0.0",
  "pidtree": "^0.5.0",
@@ -219,11 +224,13 @@
  "supertest": "^6.1.6",
  "temp-dir": "^2.0.0",
  "tomlify-j0.4": "^3.0.0",
- "tree-kill": "^1.2.2"
+ "tree-kill": "^1.2.2",
+ "typescript": "^4.4.4"
  },
  "ava": {
  "files": [
  "src/**/*.test.js",
+ "tools/**/*.test.js",
  "tests/*.test.js"
  ],
  "cache": true,

package.json

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+// @ts-check
+const { existsSync, statSync } = require('fs')
+const process = require('process')
+
+const { grey } = require('chalk')
+const execa = require('execa')
+const { sync } = require('fast-glob')
+
+const { ava } = require('../package.json')
+
+const { DependencyGraph, fileVisitor, visitorPlugins } = require('./project-graph')
+
+const getChangedFiles = async (compareTarget = 'origin/main') => {
+ const { stdout } = await execa('git', ['diff', '--name-only', 'HEAD', compareTarget])
+ return stdout.split('\n')
+}
+
+/**
+ * Get the list of affected files - if some files are touched like the package.json
+ * everything is affected.
+ * @param {string[]} changedFiles
+ * @returns {string[]}
+ */
+const getAffectedFiles = (changedFiles) => {
+ const testFiles = sync(ava.files)
+
+ // in this case all files are affected
+ if (changedFiles.includes('npm-shrinkwrap.json') || changedFiles.includes('package.json')) {
+ console.log('All files are affected based on the changeset')
+ return testFiles
+ }
+
+ const graph = new DependencyGraph()
+
+ testFiles.forEach((file) => {
+ fileVisitor(file, { graph, visitorPlugins })
+ })
+
+ return [...graph.affected(changedFiles, (file) => file.endsWith('.test.js'))]
+}
+
+/**
+ * The main function
+ * @param {string[]} args
+ */
+const main = async (args) => {
+ const changedFiles =
+ args[0] && existsSync(args[0])
+ ? args.filter((arg) => existsSync(arg) && statSync(arg).isFile())
+ : await getChangedFiles(args[0])
+
+ const affectedFiles = getAffectedFiles(changedFiles)
+
+ if (affectedFiles.length === 0) {
+ console.log('No files where affected by the changeset!')
+ return
+ }
+ console.log(`Running affected Tests: \n${grey([...affectedFiles].join(', '))}`)
+ const testRun = execa('nyc', ['-r', 'json', 'ava', ...affectedFiles], {
+ stdio: 'inherit',
+ preferLocal: true,
+ })
+
+ process.on('exit', () => {
+ testRun.kill()
+ })
+
+ try {
+ await testRun
+ } catch(error) {
+ if (error instanceof Error) {
+ console.log(error.message);
+ process.exit(1);
+ }
+ throw error;
+ }
+}
+
+// Can be invoked with two different arguments:
+// Either a list of files where all affected tests should be calculated based on:
+// $ npm run test:affected -- ./path/to/file.js ./other-file.js
+//
+// or by specifying a git diff target
+// $ npm run test:affected -- HEAD~1
+//
+// The default is when running without arguments a git diff target off 'origin/master'
+if (require.main === module) {
+ const args = process.argv.slice(2)
+ // eslint-disable-next-line promise/prefer-await-to-callbacks,promise/prefer-await-to-then
+ main(args).catch((error) => {
+ console.error(error)
+ process.exit(1)
+ })
+}
+
+module.exports = { getChangedFiles, getAffectedFiles }

tools/affected-test.js

@@ -0,0 +1,84 @@
+// @ts-check
+
+const graphviz = require('graphviz')
+
+class DependencyGraph {
+ /** @type {Map<string, Set<string>>} */
+ graph = new Map()
+
+ hasFile(fileName) {
+ return this.graph.has(fileName)
+ }
+
+ /**
+ * Adds a node to the graph
+ * @param {string} fileName
+ * @returns {void}
+ */
+ addFile(fileName) {
+ if (!this.graph.has(fileName)) {
+ this.graph.set(fileName, new Set())
+ }
+ }
+
+ addDependency(parent, child) {
+ if (!this.graph.has(parent)) {
+ this.addFile(parent)
+ }
+ if (!this.graph.has(child)) {
+ this.addFile(child)
+ }
+
+ this.graph.set(parent, this.graph.get(parent).add(child))
+ }
+
+ /**
+ * Provide a list of all affected files inside the graph based on the provided files
+ * @param {string[]} files
+ * @param {(file:string) => boolean} filterFunction
+ * @returns {Set<string>}
+ */
+ affected(files, filterFunction) {
+ const affectedFiles = new Set()
+
+ const findParents = (leaf) => {
+ if (((filterFunction && filterFunction(leaf)) || !filterFunction) && this.graph.has(leaf)) {
+ affectedFiles.add(leaf)
+ }
+
+ this.graph.forEach((value, key) => {
+ if (value.has(leaf)) {
+ if ((filterFunction && filterFunction(leaf)) || !filterFunction) {
+ affectedFiles.add(key)
+ }
+ findParents(key)
+ }
+ })
+ }
+
+ files.forEach((file) => {
+ findParents(file)
+ })
+
+ return affectedFiles
+ }
+
+ /**
+ * Visualizes a dependency graph the output is a graphviz graph
+ * that can be printed to `.to_dot()` or rendered to a png file
+ * @returns {graphviz.Graph}
+ */
+ visualize() {
+ const graph = graphviz.digraph('G')
+ this.graph.forEach((edges, node) => {
+ graph.addNode(node)
+ edges.forEach((edge) => {
+ graph.addEdge(node, edge)
+ })
+ })
+
+ return graph
+ }
+}
+
+module.exports = { DependencyGraph }