feat: send deploy validations report on secret scan

Author: aitchiss

package-lock.json

@@ -7,10 +7,12 @@ import {
  logSecretsScanSkipMessage,
  logSecretsScanSuccessMessage,
 } from '../../log/messages/core_steps.js'
+import { reportValidations } from '../../status/validations.js'
 import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js'
 
 import {
  ScanResults,
+ SecretScanResult,
  getFilePathsToScan,
  getSecretKeysToScanFor,
  groupScanResultsByKey,
@@ -20,7 +22,15 @@ import {
 
 const tracer = trace.getTracer('secrets-scanning')
 
-const coreStep: CoreStepFunction = async function ({ buildDir, logs, netlifyConfig, explicitSecretKeys, systemLog }) {
+const coreStep: CoreStepFunction = async function ({
+ buildDir,
+ logs,
+ netlifyConfig,
+ explicitSecretKeys,
+ systemLog,
+ deployId,
+ api,
+}) {
  const stepResults = {}
 
  const passedSecretKeys = (explicitSecretKeys || '').split(',')
@@ -90,6 +100,14 @@ const coreStep: CoreStepFunction = async function ({ buildDir, logs, netlifyConf
  },
  )
 
+ if (deployId) {
+ const secretScanResult: SecretScanResult = {
+ scannedFilesCount: scanResults?.scannedFilesCount ?? 0,
+ secretsScanMatches: scanResults?.matches ?? [],
+ }
+ reportValidations({ api, secretScanResult, deployId })
+ }
+
  if (!scanResults || scanResults.matches.length === 0) {
  logSecretsScanSuccessMessage(
  logs,

packages/build/src/plugins_core/secrets_scanning/index.ts

@@ -23,6 +23,11 @@ interface MatchResult {
  file: string
 }
 
+export type SecretScanResult = {
+ scannedFilesCount: number
+ secretsScanMatches: MatchResult[]
+}
+
 /**
  * Determine if the user disabled scanning via env var
  * @param env current envars

packages/build/src/plugins_core/secrets_scanning/utils.ts

@@ -29,6 +29,7 @@ export type CoreStepFunctionArgs = {
  explicitSecretKeys: $TSFixme
 
  buildbotServerSocket: $TSFixme
+ api?: unknown
 }
 
 export type CoreStepFunction = (args: CoreStepFunctionArgs) => Promise<object>

packages/build/src/plugins_core/types.ts

@@ -0,0 +1,19 @@
+import { SecretScanResult } from '../plugins_core/secrets_scanning/utils.js'
+
+// Reports any validations completed on the deploy to the API
+export const reportValidations = async function ({
+ api,
+ secretScanResult,
+ deployId,
+}: {
+ api: unknown
+ secretScanResult: SecretScanResult
+ deployId: string
+}) {
+ try {
+ // @ts-expect-error API type is not defined
+ api.updateDeployValidations({ deploy_id: deployId, body: { secrets_scan: secretScanResult } })
+ } catch {
+ // Noop
+ }
+}

packages/build/src/status/validations.ts

@@ -40,6 +40,7 @@ export const fireCoreStep = async function ({
  edgeFunctionsBootstrapURL,
  deployId,
  outputFlusher,
+ api,
 }) {
  const logsA = outputFlusher ? addOutputFlusher(logs, outputFlusher) : logs
 
@@ -52,6 +53,7 @@ export const fireCoreStep = async function ({
  tags,
  metrics,
  } = await coreStep({
+ api,
  configPath,
  outputConfigPath,
  buildDir,

packages/build/src/steps/core_step.ts

@@ -184,6 +184,7 @@ export const runStep = async function ({
  explicitSecretKeys,
  edgeFunctionsBootstrapURL,
  deployId,
+ api,
  })
 
  const newValues = await getStepReturn({
@@ -346,6 +347,7 @@ const tFireStep = function ({
  edgeFunctionsBootstrapURL,
  deployId,
  extensionMetadata,
+ api,
 }) {
  if (coreStep !== undefined) {
  return fireCoreStep({
@@ -383,6 +385,7 @@ const tFireStep = function ({
  explicitSecretKeys,
  edgeFunctionsBootstrapURL,
  deployId,
+ api,
  })
  }
 

packages/build/src/steps/run_step.ts

@@ -1,4 +1,4 @@
-# Snapshot report for `tests/secrets_scanning/tests.js`
+# Snapshot report for `packages/build/tests/secrets_scanning/tests.js`
 
 The actual snapshot is saved in `tests.js.snap`.
 
@@ -276,6 +276,154 @@ Generated by [AVA](https://avajs.dev).
  ␊
  (Netlify Build completed in 1ms)`
 
+## secrets scanning, should fail build when it finds secrets in the src and build output and report to API
+
+> Snapshot 1
+
+ `␊
+ Netlify Build ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ > Version␊
+  @netlify/build 1.0.0␊
+ ␊
+ > Flags␊
+  debug: false␊
+ ␊
+ > Current directory␊
+  packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_set_non_empty␊
+ ␊
+ > Config file␊
+  packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_set_non_empty/netlify.toml␊
+ ␊
+ > Context␊
+  production␊
+ ␊
+ build.command from netlify.toml ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ $ cp -r ./src/. ./dist␊
+ ␊
+ (build.command completed in 1ms)␊
+ ␊
+ Scanning for secrets in code and build output. ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ ␊
+ > Scanning complete. 14 file(s) scanned. Secrets scanning found 32 instance(s) of secrets in build output or repo code.␊
+ ␊
+ Secret env var "ENV_VAR_1"'s value detected:␊
+  found value at line 12 in dist/static-files/static-a.txt␊
+  found value at line 6 in netlify.toml␊
+  found value at line 12 in src/static-files/static-a.txt␊
+ Secret env var "ENV_VAR_2"'s value detected:␊
+  found value at line 1 in dist/some-file.txt␊
+  found value at line 1 in dist/static-files/static-a.txt␊
+  found value at line 6 in dist/static-files/static-a.txt␊
+  found value at line 7 in netlify.toml␊
+  found value at line 1 in src/some-file.txt␊
+  found value at line 1 in src/static-files/static-a.txt␊
+  found value at line 6 in src/static-files/static-a.txt␊
+ Secret env var "ENV_VAR_3"'s value detected:␊
+  found value at line 14 in dist/static-files/static-a.txt␊
+  found value at line 16 in dist/static-files/static-a.txt␊
+  found value at line 1 in dist/static-files/static-c.txt␊
+  found value at line 8 in netlify.toml␊
+  found value at line 14 in src/static-files/static-a.txt␊
+  found value at line 16 in src/static-files/static-a.txt␊
+  found value at line 1 in src/static-files/static-c.txt␊
+ Secret env var "ENV_VAR_4"'s value detected:␊
+  found value at line 20 in dist/static-files/static-a.txt␊
+  found value at line 9 in netlify.toml␊
+  found value at line 20 in src/static-files/static-a.txt␊
+ Secret env var "ENV_VAR_MULTILINE_A"'s value detected:␊
+  found value at line 17 in dist/static-files/static-c.txt␊
+  found value at line 38 in dist/static-files/static-c.txt␊
+  found value at line 1 in dist/static-files/static-d.txt␊
+  found value at line 15 in netlify.toml␊
+  found value at line 17 in src/static-files/static-c.txt␊
+  found value at line 38 in src/static-files/static-c.txt␊
+  found value at line 1 in src/static-files/static-d.txt␊
+ Secret env var "ENV_VAR_MULTILINE_B"'s value detected:␊
+  found value at line 4 in dist/static-files/static-d.txt␊
+  found value at line 1 in dist/static-files/static-e.txt␊
+  found value at line 21 in netlify.toml␊
+  found value at line 4 in src/static-files/static-d.txt␊
+  found value at line 1 in src/static-files/static-e.txt␊
+ ␊
+ To prevent exposing secrets, the build will fail until these secret values are not found in build output or repo files.␊
+ If these are expected, use SECRETS_SCAN_OMIT_PATHS, SECRETS_SCAN_OMIT_KEYS, or SECRETS_SCAN_ENABLED to prevent detecting.␊
+ For more information on secrets scanning, see the Netlify Docs: https://ntl.fyi/configure-secrets-scanning␊
+ ␊
+ Secrets scanning detected secrets in files during build. ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+  Error message␊
+  Secrets scanning found secrets in build.␊
+ ␊
+  Resolved config␊
+  build:␊
+  command: cp -r ./src/. ./dist␊
+  commandOrigin: config␊
+  environment:␊
+  - ENV_VAR_1␊
+  - ENV_VAR_2␊
+  - ENV_VAR_3␊
+  - ENV_VAR_4␊
+  - ENV_VAR_5␊
+  - ENV_VAR_6␊
+  - ENV_VAR_7␊
+  - NOT_SECRET_VAL␊
+  - ENV_VAR_MULTILINE_A␊
+  - ENV_VAR_MULTILINE_B␊
+  - ENV_VAR_MULTI_NOT_SECRET␊
+  publish: packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_set_non_empty/dist␊
+  publishOrigin: config`
+
+## secrets scanning should report success to API when no secrets are found
+
+> Snapshot 1
+
+ `␊
+ Netlify Build ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ > Version␊
+  @netlify/build 1.0.0␊
+ ␊
+ > Flags␊
+  debug: false␊
+ ␊
+ > Current directory␊
+  packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches␊
+ ␊
+ > Config file␊
+  packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches/netlify.toml␊
+ ␊
+ > Context␊
+  production␊
+ ␊
+ build.command from netlify.toml ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ $ cp -r ./src/static-files ./dist␊
+ ␊
+ (build.command completed in 1ms)␊
+ ␊
+ Scanning for secrets in code and build output. ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ SECRETS_SCAN_OMIT_PATHS override option set to: netlify.toml␊
+ ␊
+ Secrets scanning complete. 4 file(s) scanned. No secrets detected in build output or repo code!␊
+ ␊
+ (Secrets scanning completed in 1ms)␊
+ ␊
+ Netlify Build Complete ␊
+ ────────────────────────────────────────────────────────────────␊
+ ␊
+ (Netlify Build completed in 1ms)`
+
 ## secrets scanning, should fail build when it finds secrets in the src and build output
 
 > Snapshot 1

packages/build/tests/secrets_scanning/snapshots/tests.js.md

@@ -60,15 +60,22 @@ test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_PA
  t.assert(normalizeOutput(output).includes('found value at line 1 in src/static-files/notsafefile.js'))
 })
 
-test('secrets scanning, should fail build when it finds secrets in the src and build output', async (t) => {
- const output = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty')
+test('secrets scanning should report success to API when no secrets are found', async (t) => {
+ const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_no_matches')
  .withFlags({
  debug: false,
- explicitSecretKeys:
- 'ENV_VAR_MULTILINE_A,ENV_VAR_1,ENV_VAR_2,ENV_VAR_3,ENV_VAR_4,ENV_VAR_5,ENV_VAR_6,ENV_VAR_MULTILINE_B',
+ explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_2',
+ deployId: 'test',
+ token: 'test',
  })
- .runWithBuild()
- t.snapshot(normalizeOutput(output))
+ .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+ console.log({ requests })
+ t.true(requests.length === 1)
+ const request = requests[0]
+ t.is(request.method, 'PATCH')
+ t.is(request.url, '/api/v1/deploys/test/validations_report')
+ t.deepEqual(request.body, { secrets_scan: { scannedFilesCount: 4, secretsScanMatches: [] } })
 })
 
 test('secrets scanning failure should produce an user error', async (t) => {
@@ -83,6 +90,30 @@ test('secrets scanning failure should produce an user error', async (t) => {
  t.is(severityCode, 2)
 })
 
+test('secrets scanning should report failure to API when secrets are found', async (t) => {
+ const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty')
+ .withFlags({
+ debug: false,
+ explicitSecretKeys:
+ 'ENV_VAR_MULTILINE_A,ENV_VAR_1,ENV_VAR_2,ENV_VAR_3,ENV_VAR_4,ENV_VAR_5,ENV_VAR_6,ENV_VAR_MULTILINE_B',
+ deployId: 'test',
+ token: 'test',
+ })
+ .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+ t.true(requests.length === 1)
+ const request = requests[0]
+ t.is(request.method, 'PATCH')
+ t.is(request.url, '/api/v1/deploys/test/validations_report')
+ t.is(request.body.secrets_scan.scannedFilesCount, 14)
+ t.is(request.body.secrets_scan.secretsScanMatches.length, 32)
+ t.deepEqual(request.body.secrets_scan.secretsScanMatches[0], {
+ file: 'netlify.toml',
+ lineNumber: 6,
+ key: 'ENV_VAR_1',
+ })
+})
+
 test('secrets scanning, should not fail if the secrets values are not detected in the build output', async (t) => {
  const output = await new Fixture('./fixtures/src_scanning_env_vars_no_matches')
  .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_2' })