fix: EMFILE errors (#18313)

* fix: EMFILE errors fixes #18301 * Move catch handler * Add intentional EMFILE failure * Use actual limit on Linux systems in test * Adjust emfile test limit * Fix linting error * Fix test for MacOS * Up MacOS limit in test * Move tmp file output directory * Update .gitignore Co-authored-by: Francesco Trotta <github@fasttime.org> --------- Co-authored-by: Francesco Trotta <github@fasttime.org>

Author: nzakas

.github/workflows/ci.yml

@@ -65,6 +65,8 @@ jobs:
  run: node Makefile mocha
  - name: Fuzz Test
  run: node Makefile fuzz
+ - name: Test EMFILE Handling
+ run: npm run test:emfile
 
  test_on_browser:
  name: Browser Test

lib/eslint/eslint.js

@@ -42,6 +42,7 @@ const {
 const { pathToFileURL } = require("url");
 const { FlatConfigArray } = require("../config/flat-config-array");
 const LintResultCache = require("../cli-engine/lint-result-cache");
+const { Retrier } = require("@humanwhocodes/retry");
 
 /*
  * This is necessary to allow overwriting writeFile for testing purposes.
@@ -851,6 +852,8 @@ class ESLint {
  errorOnUnmatchedPattern
  });
  const controller = new AbortController();
+ const retryCodes = new Set(["ENFILE", "EMFILE"]);
+ const retrier = new Retrier(error => retryCodes.has(error.code));
 
  debug(`${filePaths.length} files found in: ${Date.now() - startTime}ms`);
 
@@ -919,7 +922,7 @@ class ESLint {
  fixer = message => shouldMessageBeFixed(message, config, fixTypesSet) && originalFix(message);
  }
 
- return fs.readFile(filePath, { encoding: "utf8", signal: controller.signal })
+ return retrier.retry(() => fs.readFile(filePath, { encoding: "utf8", signal: controller.signal })
  .then(text => {
 
  // fail immediately if an error occurred in another file
@@ -949,11 +952,11 @@ class ESLint {
  }
 
  return result;
- }).catch(error => {
+ }))
+ .catch(error => {
  controller.abort(error);
  throw error;
  });
-
  })
  );
 

package.json

@@ -33,7 +33,8 @@
  "test:browser": "node Makefile.js wdio",
  "test:cli": "mocha",
  "test:fuzz": "node Makefile.js fuzz",
- "test:performance": "node Makefile.js perf"
+ "test:performance": "node Makefile.js perf",
+ "test:emfile": "node tools/check-emfile-handling.js"
  },
  "gitHooks": {
  "pre-commit": "lint-staged"
@@ -71,6 +72,7 @@
  "@eslint/js": "9.0.0",
  "@humanwhocodes/config-array": "^0.12.3",
  "@humanwhocodes/module-importer": "^1.0.1",
+ "@humanwhocodes/retry": "^0.2.3",
  "@nodelib/fs.walk": "^1.2.8",
  "ajv": "^6.12.4",
  "chalk": "^4.0.0",

tests/fixtures/emfile/eslint.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+ rules: {
+ "no-unused-vars": "error"
+ }
+};

tools/check-emfile-handling.js

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview A utility to test that ESLint doesn't crash with EMFILE/ENFILE errors.
+ * @author Nicholas C. Zakas
+ */
+
+"use strict";
+
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const fs = require("fs");
+const { readFile } = require("fs/promises");
+const { execSync } = require("child_process");
+const os = require("os");
+
+//------------------------------------------------------------------------------
+// Helpers
+//------------------------------------------------------------------------------
+
+const OUTPUT_DIRECTORY = "tmp/emfile-check";
+const CONFIG_DIRECTORY = "tests/fixtures/emfile";
+
+/*
+ * Every operating system has a different limit for the number of files that can
+ * be opened at once. This number is meant to be larger than the default limit
+ * on most systems.
+ *
+ * Linux systems typically start at a count of 1024 and may be increased to 4096.
+ * MacOS Sonoma v14.4 has a limit of 10496.
+ * Windows has no hard limit but may be limited by available memory.
+ */
+const DEFAULT_FILE_COUNT = 15000;
+let FILE_COUNT = DEFAULT_FILE_COUNT;
+
+// if the platform isn't windows, get the ulimit to see what the actual limit is
+if (os.platform() !== "win32") {
+ try {
+ FILE_COUNT = parseInt(execSync("ulimit -n").toString().trim(), 10) + 1;
+
+ console.log(`Detected Linux file limit of ${FILE_COUNT}.`);
+
+ // if we're on a Mac, make sure the limit isn't high enough to cause a call stack error
+ if (os.platform() === "darwin") {
+ FILE_COUNT = Math.min(FILE_COUNT, 100000);
+ }
+ } catch {
+
+ // ignore error and use default
+ }
+}
+
+/**
+ * Generates files in a directory.
+ * @returns {void}
+ */
+function generateFiles() {
+
+ fs.mkdirSync(OUTPUT_DIRECTORY, { recursive: true });
+
+ for (let i = 0; i < FILE_COUNT; i++) {
+ const fileName = `file_${i}.js`;
+ const fileContent = `// This is file ${i}`;
+
+ fs.writeFileSync(`${OUTPUT_DIRECTORY}/${fileName}`, fileContent);
+ }
+
+}
+
+/**
+ * Generates an EMFILE error by reading all files in the output directory.
+ * @returns {Promise<Buffer[]>} A promise that resolves with the contents of all files.
+ */
+function generateEmFileError() {
+ return Promise.all(
+ Array.from({ length: FILE_COUNT }, (_, i) => {
+ const fileName = `file_${i}.js`;
+
+ return readFile(`${OUTPUT_DIRECTORY}/${fileName}`);
+ })
+ );
+}
+
+//------------------------------------------------------------------------------
+// Main
+//------------------------------------------------------------------------------
+
+console.log(`Generating ${FILE_COUNT} files in ${OUTPUT_DIRECTORY}...`);
+generateFiles();
+
+console.log("Running ESLint...");
+execSync(`node bin/eslint.js ${OUTPUT_DIRECTORY} -c ${CONFIG_DIRECTORY}/eslint.config.js`, { stdio: "inherit" });
+console.log("✅ No errors encountered running ESLint.");
+
+console.log("Checking that this number of files would cause an EMFILE error...");
+generateEmFileError()
+ .then(() => {
+ throw new Error("EMFILE error not encountered.");
+ })
+ .catch(error => {
+ if (error.code === "EMFILE") {
+ console.log("✅ EMFILE error encountered:", error.message);
+ } else if (error.code === "ENFILE") {
+ console.log("✅ ENFILE error encountered:", error.message);
+ } else {
+ console.error("❌ Unexpected error encountered:", error.message);
+ throw error;
+ }
+ });