Improve run-test262

- add -t to show timings - add -C to select compact progress meter - default to compact progress meter if not attached to console - set agent stack size to 2MB - compute module filename relative to current path - ignore `testdir` for -d and -f options - return non zero status on errors changes

Author: chqrlie

Makefile

@@ -488,13 +488,13 @@ test2o test2o-32 test2o-update:
 else
 # ES5 tests (obsolete)
 test2o: run-test262
-time ./run-test262 -m -c test262o.conf
+time ./run-test262 -t -m -c test262o.conf
 
 test2o-32: run-test262-32
-time ./run-test262-32 -m -c test262o.conf
+time ./run-test262-32 -t -m -c test262o.conf
 
 test2o-update: run-test262
-./run-test262 -u -c test262o.conf
+./run-test262 -t -u -c test262o.conf
 endif
 
 ifeq ($(wildcard test262o/tests.txt),)
@@ -503,19 +503,19 @@ test2 test2-32 test2-update test2-default test2-check:
 else
 # Test262 tests
 test2-default: run-test262
-time ./run-test262 -m -c test262.conf
+time ./run-test262 -t -m -c test262.conf
 
 test2: run-test262
-time ./run-test262 -m -c test262.conf -a
+time ./run-test262 -t -m -c test262.conf -a
 
 test2-32: run-test262-32
-time ./run-test262-32 -m -c test262.conf -a
+time ./run-test262-32 -t -m -c test262.conf -a
 
 test2-update: run-test262
-./run-test262 -u -c test262.conf -a
+./run-test262 -t -u -c test262.conf -a
 
 test2-check: run-test262
-time ./run-test262 -m -c test262.conf -E -a
+time ./run-test262 -t -m -c test262.conf -E -a
 endif
 
 testall: all test microbench test2o test2

run-test262.c

@@ -63,6 +63,8 @@ enum test_mode_t {
  TEST_STRICT, /* run tests as strict, skip nostrict tests */
  TEST_ALL, /* run tests in both strict and nostrict, unless restricted by spec */
 } test_mode = TEST_DEFAULT_NOSTRICT;
+int compact;
+int show_timings;
 int skip_async;
 int skip_module;
 int new_style;
@@ -530,6 +532,7 @@ static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
 {
  const char *script;
  Test262Agent *agent;
+ pthread_attr_t attr;
 
  if (JS_GetContextOpaque(ctx) != NULL)
  return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
@@ -544,7 +547,12 @@ static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
  agent->script = strdup(script);
  JS_FreeCString(ctx, script);
  list_add_tail(&agent->link, &agent_list);
- pthread_create(&agent->tid, NULL, agent_start, agent);
+ pthread_attr_init(&attr);
+ // musl libc gives threads 80 kb stacks, much smaller than
+ // JS_DEFAULT_STACK_SIZE (256 kb)
+ pthread_attr_setstacksize(&attr, 2 << 20); // 2 MB, glibc default
+ pthread_create(&agent->tid, &attr, agent_start, agent);
+ pthread_attr_destroy(&attr);
  return JS_UNDEFINED;
 }
 
@@ -813,6 +821,19 @@ static JSModuleDef *js_module_loader_test(JSContext *ctx,
  uint8_t *buf;
  JSModuleDef *m;
  JSValue func_val;
+ char *filename, *slash, path[1024];
+
+ // interpret import("bar.js") from path/to/foo.js as
+ // import("path/to/bar.js") but leave import("./bar.js") untouched
+ filename = opaque;
+ if (!strchr(module_name, '/')) {
+ slash = strrchr(filename, '/');
+ if (slash) {
+ snprintf(path, sizeof(path), "%.*s/%s",
+ (int)(slash - filename), filename, module_name);
+ module_name = path;
+ }
+ }
 
  buf = js_load_file(ctx, &buf_len, module_name);
  if (!buf) {
@@ -910,7 +931,7 @@ void update_exclude_dirs(void)
  lp->count = count;
 }
 
-void load_config(const char *filename)
+void load_config(const char *filename, const char *ignore)
 {
  char buf[1024];
  FILE *f;
@@ -965,6 +986,10 @@ void load_config(const char *filename)
  printf("%s:%d: syntax error\n", filename, lineno);
  continue;
  }
+ if (strstr(ignore, p)) {
+ printf("%s:%d: ignoring %s=%s\n", filename, lineno, p, q);
+ continue;
+ }
  if (str_equal(p, "style")) {
  new_style = str_equal(q, "new");
  continue;
@@ -1540,7 +1565,7 @@ int run_test_buf(const char *filename, const char *harness, namelist_t *ip,
  JS_SetCanBlock(rt, can_block);
 
  /* loader for ES6 modules */
- JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
+ JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename);
 
  add_helpers(ctx);
 
@@ -1656,7 +1681,7 @@ int run_test(const char *filename, int index)
  /* XXX: should extract the phase */
  char *q = find_tag(p, "type:", &state);
  if (q) {
- while (isspace(*q))
+ while (isspace((unsigned char)*q))
  q++;
  error_type = strdup_len(q, strcspn(q, " \n"));
  }
@@ -1841,7 +1866,7 @@ int run_test262_harness_test(const char *filename, BOOL is_module)
  JS_SetCanBlock(rt, can_block);
 
  /* loader for ES6 modules */
- JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
+ JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename);
 
  add_helpers(ctx);
 
@@ -1900,9 +1925,27 @@ void show_progress(int force) {
  clock_t t = clock();
  if (force || !last_clock || (t - last_clock) > CLOCKS_PER_SEC / 20) {
  last_clock = t;
- /* output progress indicator: erase end of line and return to col 0 */
- fprintf(stderr, "%d/%d/%d\033[K\r",
- test_failed, test_count, test_skipped);
+ if (compact) {
+ static int last_test_skipped;
+ static int last_test_failed;
+ static int dots;
+ char c = '.';
+ if (test_skipped > last_test_skipped)
+ c = '-';
+ if (test_failed > last_test_failed)
+ c = '!';
+ last_test_skipped = test_skipped;
+ last_test_failed = test_failed;
+ fputc(c, stderr);
+ if (force || ++dots % 60 == 0) {
+ fprintf(stderr, " %d/%d/%d\n",
+ test_failed, test_count, test_skipped);
+ }
+ } else {
+ /* output progress indicator: erase end of line and return to col 0 */
+ fprintf(stderr, "%d/%d/%d\033[K\r",
+ test_failed, test_count, test_skipped);
+ }
  fflush(stderr);
  }
 }
@@ -1953,6 +1996,8 @@ void help(void)
  "-N run test prepared by test262-harness+eshost\n"
  "-s run tests in strict mode, skip @nostrict tests\n"
  "-E only run tests from the error file\n"
+ "-C use compact progress indicator\n"
+ "-t show timings\n"
  "-u update error file\n"
  "-v verbose: output error messages\n"
  "-T duration display tests taking more than 'duration' ms\n"
@@ -1979,14 +2024,29 @@ int main(int argc, char **argv)
  BOOL is_dir_list;
  BOOL only_check_errors = FALSE;
  const char *filename;
+ const char *ignore = "";
  BOOL is_test262_harness = FALSE;
  BOOL is_module = FALSE;
+ clock_t clocks;
 
 #if !defined(_WIN32)
+ compact = !isatty(STDERR_FILENO);
  /* Date tests assume California local time */
  setenv("TZ", "America/Los_Angeles", 1);
 #endif
 
+ optind = 1;
+ while (optind < argc) {
+ char *arg = argv[optind];
+ if (*arg != '-')
+ break;
+ optind++;
+ if (strstr("-c -d -e -x -f -r -E -T", arg))
+ optind++;
+ if (strstr("-d -f", arg))
+ ignore = "testdir"; // run only the tests from -d or -f
+ }
+
  /* cannot use getopt because we want to pass the command line to
  the script */
  optind = 1;
@@ -2006,12 +2066,16 @@ int main(int argc, char **argv)
  test_mode = TEST_STRICT;
  } else if (str_equal(arg, "-a")) {
  test_mode = TEST_ALL;
+ } else if (str_equal(arg, "-t")) {
+ show_timings++;
  } else if (str_equal(arg, "-u")) {
  update_errors++;
  } else if (str_equal(arg, "-v")) {
  verbose++;
+ } else if (str_equal(arg, "-C")) {
+ compact = 1;
  } else if (str_equal(arg, "-c")) {
- load_config(get_opt_arg(arg, argv[optind++]));
+ load_config(get_opt_arg(arg, argv[optind++]), ignore);
  } else if (str_equal(arg, "-d")) {
  enumerate_tests(get_opt_arg(arg, argv[optind++]));
  } else if (str_equal(arg, "-e")) {
@@ -2042,7 +2106,7 @@ int main(int argc, char **argv)
  if (is_test262_harness) {
  return run_test262_harness_test(argv[optind], is_module);
  }
-
+
  error_out = stdout;
  if (error_filename) {
  error_file = load_file(error_filename, NULL);
@@ -2062,8 +2126,10 @@ int main(int argc, char **argv)
 
  update_exclude_dirs();
 
+ clocks = clock();
+
  if (is_dir_list) {
- if (optind < argc && !isdigit(argv[optind][0])) {
+ if (optind < argc && !isdigit((unsigned char)argv[optind][0])) {
  filename = argv[optind++];
  namelist_load(&test_list, filename);
  }
@@ -2098,6 +2164,8 @@ int main(int argc, char **argv)
  }
  }
 
+ clocks = clock() - clocks;
+
  if (dump_memory) {
  if (dump_memory > 1 && stats_count > 1) {
  printf("\nMininum memory statistics for %s:\n\n", stats_min_filename);
@@ -2126,6 +2194,8 @@ int main(int argc, char **argv)
  fprintf(stderr, ", %d fixed", fixed_errors);
  }
  fprintf(stderr, "\n");
+ if (show_timings)
+ fprintf(stderr, "Total time: %.3fs\n", (double)clocks / CLOCKS_PER_SEC);
  }
 
  if (error_out && error_out != stdout) {
@@ -2141,5 +2211,6 @@ int main(int argc, char **argv)
  free(harness_exclude);
  free(error_file);
 
- return 0;
+ /* Signal that the error file is out of date. */
+ return new_errors || changed_errors || fixed_errors;
 }