refactor(bench press): rename metrics and adapt them to the features of the browser

* Rename metrics, add `Time` suffix to all so that they are more consistent * iOS does not give us `gc` metrics, so they should not be reported * Rename `scriptMicroAvg` into `microScriptTimeAvg` * Rename previous `script` metric into `pureScriptTime` metric, and keep `scriptTime` metric as the overall time, so that we still have a shared metric across devices independent of the supported browser features * `microScriptTimeAvg` is now based on overall `scriptTime`, including gc and render time. * Move more shared DI tokens into `common_options` (previously `sample_options`). Closes angular#930

Author: tbosch

modules/benchpress/benchpress.es6

@@ -1,14 +1,18 @@
 import { bind } from 'angular2/di';
-import { JsonFileReporter } from './common';
+import { Options } from './common';
 
 export * from './common';
 export { SeleniumWebDriverAdapter } from './src/webdriver/selenium_webdriver_adapter';
 
 var fs = require('fs');
 
-// Note: Can't do the `require` call in a facade as it can't be loaded into the browser!
-JsonFileReporter.BINDINGS.push(
- bind(JsonFileReporter.WRITE_FILE).toValue(writeFile)
+// TODO(tbosch): right now we bind the `writeFile` method
+// in benchpres/benchpress.es6. This does not work for Dart,
+// find another way...
+// Note: Can't do the `require` call in a facade as it can't be loaded into the browser
+// for our unit tests via karma.
+Options.DEFAULT_BINDINGS.push(
+ bind(Options.WRITE_FILE).toValue(writeFile)
 );
 
 function writeFile(filename, content) {

modules/benchpress/common.js

@@ -2,7 +2,7 @@ export { Sampler, SampleState } from './src/sampler';
 export { Metric } from './src/metric';
 export { Validator } from './src/validator';
 export { Reporter } from './src/reporter';
-export { WebDriverExtension } from './src/web_driver_extension';
+export { WebDriverExtension, PerfLogFeatures } from './src/web_driver_extension';
 export { WebDriverAdapter } from './src/web_driver_adapter';
 export { SizeValidator } from './src/validator/size_validator';
 export { RegressionSlopeValidator } from './src/validator/regression_slope_validator';
@@ -13,7 +13,7 @@ export { PerflogMetric } from './src/metric/perflog_metric';
 export { ChromeDriverExtension } from './src/webdriver/chrome_driver_extension';
 export { IOsDriverExtension } from './src/webdriver/ios_driver_extension';
 export { Runner } from './src/runner';
-export { Options } from './src/sample_options';
+export { Options } from './src/common_options';
 export { MeasureValues } from './src/measure_values';
 export { MultiMetric } from './src/metric/multi_metric';
 export { MultiReporter } from './src/reporter/multi_reporter';

modules/benchpress/src/sample_options.js renamed to modules/benchpress/src/common_options.js

@@ -1,6 +1,8 @@
 import { bind, OpaqueToken } from 'angular2/di';
+import { DateWrapper } from 'angular2/src/facade/lang';
 
 export class Options {
+ static get DEFAULT_BINDINGS() { return _DEFAULT_BINDINGS; }
  // TODO(tbosch): use static initializer when our transpiler supports it
  static get SAMPLE_ID() { return _SAMPLE_ID; }
  // TODO(tbosch): use static initializer when our transpiler supports it
@@ -23,6 +25,10 @@ export class Options {
  * Used for micro benchmarks.
  **/
  static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; }
+ // TODO(tbosch): use static initializer when our transpiler supports it
+ static get NOW() { return _NOW; }
+ // TODO(tbosch): use static values when our transpiler supports them
+ static get WRITE_FILE() { return _WRITE_FILE; }
 }
 
 var _SAMPLE_ID = new OpaqueToken('Options.sampleId');
@@ -34,3 +40,13 @@ var _EXECUTE = new OpaqueToken('Options.execute');
 var _CAPABILITIES = new OpaqueToken('Options.capabilities');
 var _USER_AGENT = new OpaqueToken('Options.userAgent');
 var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations');
+var _NOW = new OpaqueToken('Options.now');
+var _WRITE_FILE = new OpaqueToken('Options.writeFile');
+
+var _DEFAULT_BINDINGS = [
+ bind(_DEFAULT_DESCRIPTION).toValue({}),
+ bind(_SAMPLE_DESCRIPTION).toValue({}),
+ bind(_FORCE_GC).toValue(false),
+ bind(_PREPARE).toValue(false),
+ bind(_NOW).toValue( () => DateWrapper.now() )
+];

modules/benchpress/src/metric/perflog_metric.js

@@ -3,9 +3,9 @@ import { isPresent, isBlank, int, BaseException, StringWrapper, Math } from 'ang
 import { ListWrapper, StringMap, StringMapWrapper } from 'angular2/src/facade/collection';
 import { bind, OpaqueToken } from 'angular2/di';
 
-import { WebDriverExtension } from '../web_driver_extension';
+import { WebDriverExtension, PerfLogFeatures } from '../web_driver_extension';
 import { Metric } from '../metric';
-import { Options } from '../sample_options';
+import { Options } from '../common_options';
 
 /**
  * A metric that reads out the performance log
@@ -21,6 +21,7 @@ export class PerflogMetric extends Metric {
  _measureCount:int;
  _setTimeout:Function;
  _microIterations:int;
+ _perfLogFeatures:PerfLogFeatures;
 
  /**
  * @param driverExtension
@@ -35,19 +36,24 @@ export class PerflogMetric extends Metric {
  this._measureCount = 0;
  this._setTimeout = setTimeout;
  this._microIterations = microIterations;
+ this._perfLogFeatures = driverExtension.perfLogFeatures();
  }
 
  describe():StringMap {
  var res = {
- 'script': 'script execution time in ms',
- 'render': 'render time in ms',
- 'gcTime': 'gc time in ms',
- 'gcAmount': 'gc amount in kbytes',
- 'majorGcTime': 'time of major gcs in ms',
- 'majorGcAmount': 'amount of major gcs in kbytes'
+ 'scriptTime': 'script execution time in ms, including gc and render',
+ 'pureScriptTime': 'script execution time in ms, without gc nor render'
  };
+ if (this._perfLogFeatures.render) {
+ res['renderTime'] = 'render time in and ouside of script in ms';
+ }
+ if (this._perfLogFeatures.gc) {
+ res['gcTime'] = 'gc time in and ouside of script in ms';
+ res['gcAmount'] = 'gc amount in kbytes';
+ res['majorGcTime'] = 'time of major gcs in ms';
+ }
  if (this._microIterations > 0) {
- res['scriptMicroAvg'] = 'average script time for a micro iteration';
+ res['microScriptTimeAvg'] = 'average script time for a micro iteration';
  }
  return res;
  }
@@ -120,17 +126,22 @@ export class PerflogMetric extends Metric {
 
  _aggregateEvents(events, markName) {
  var result = {
- 'script': 0,
- 'render': 0,
- 'gcTime': 0,
- 'gcAmount': 0,
- 'majorGcTime': 0,
- 'majorGcAmount': 0
+ 'scriptTime': 0,
+ 'pureScriptTime': 0
  };
+ if (this._perfLogFeatures.gc) {
+ result['gcTime'] = 0;
+ result['majorGcTime'] = 0;
+ result['gcAmount'] = 0;
+ }
+ if (this._perfLogFeatures.render) {
+ result['renderTime'] = 0;
+ }
 
  var markStartEvent = null;
  var markEndEvent = null;
  var gcTimeInScript = 0;
+ var renderTimeInScript = 0;
 
  var intervalStarts = {};
  events.forEach( (event) => {
@@ -149,26 +160,30 @@ export class PerflogMetric extends Metric {
  var duration = event['ts'] - startEvent['ts'];
  intervalStarts[name] = null;
  if (StringWrapper.equals(name, 'gc')) {
- var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000;
  result['gcTime'] += duration;
+ var amount = (startEvent['args']['usedHeapSize'] - event['args']['usedHeapSize']) / 1000;
  result['gcAmount'] += amount;
  var majorGc = event['args']['majorGc'];
  if (isPresent(majorGc) && majorGc) {
  result['majorGcTime'] += duration;
- result['majorGcAmount'] += amount;
  }
  if (isPresent(intervalStarts['script'])) {
  gcTimeInScript += duration;
  }
- } else if (StringWrapper.equals(name, 'script') || StringWrapper.equals(name, 'render')) {
- result[name] += duration;
+ } else if (StringWrapper.equals(name, 'render')) {
+ result['renderTime'] += duration;
+ if (isPresent(intervalStarts['script'])) {
+ renderTimeInScript += duration;
+ }
+ } else if (StringWrapper.equals(name, 'script')) {
+ result['scriptTime'] += duration;
  }
  }
  }
  });
- result['script'] -= gcTimeInScript;
+ result['pureScriptTime'] = result['scriptTime'] - gcTimeInScript - renderTimeInScript;
  if (this._microIterations > 0) {
- result['scriptMicroAvg'] = result['script'] / this._microIterations;
+ result['microScriptTimeAvg'] = result['scriptTime'] / this._microIterations;
  }
  return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null;
  }
@@ -183,7 +198,8 @@ var _MARK_NAME_PREFIX = 'benchpress';
 var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout');
 var _BINDINGS = [
  bind(PerflogMetric).toFactory(
- (driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations),
+ (driverExtension, setTimeout, microIterations) =>
+ new PerflogMetric(driverExtension, setTimeout, microIterations),
  [WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS]
  ),
  bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ),

modules/benchpress/src/reporter/json_file_reporter.js

@@ -7,29 +7,27 @@ import { bind, OpaqueToken } from 'angular2/di';
 import { Reporter } from '../reporter';
 import { SampleDescription } from '../sample_description';
 import { MeasureValues } from '../measure_values';
+import { Options } from '../common_options';
 
 /**
  * A reporter that writes results into a json file.
- * TODO(tbosch): right now we bind the `writeFile` method
- * in benchpres/benchpress.es6. This does not work for Dart,
- * find another way...
  */
 export class JsonFileReporter extends Reporter {
- // TODO(tbosch): use static values when our transpiler supports them
- static get WRITE_FILE() { return _WRITE_FILE; }
  // TODO(tbosch): use static values when our transpiler supports them
  static get PATH() { return _PATH; }
  static get BINDINGS() { return _BINDINGS; }
 
  _writeFile:Function;
  _path:string;
  _description:SampleDescription;
+ _now:Function;
 
- constructor(sampleDescription, path, writeFile) {
+ constructor(sampleDescription, path, writeFile, now) {
  super();
  this._description = sampleDescription;
  this._path = path;
  this._writeFile = writeFile;
+ this._now = now;
  }
 
  reportMeasureValues(measureValues:MeasureValues):Promise {
@@ -42,17 +40,16 @@ export class JsonFileReporter extends Reporter {
  'completeSample': completeSample,
  'validSample': validSample
  });
- var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(DateWrapper.now())}.json`;
+ var filePath = `${this._path}/${this._description.id}_${DateWrapper.toMillis(this._now())}.json`;
  return this._writeFile(filePath, content);
  }
 }
 
-var _WRITE_FILE = new OpaqueToken('JsonFileReporter.writeFile');
 var _PATH = new OpaqueToken('JsonFileReporter.path');
 var _BINDINGS = [
  bind(JsonFileReporter).toFactory(
- (sampleDescription, path, writeFile) => new JsonFileReporter(sampleDescription, path, writeFile),
- [SampleDescription, _PATH, _WRITE_FILE]
+ (sampleDescription, path, writeFile, now) => new JsonFileReporter(sampleDescription, path, writeFile, now),
+ [SampleDescription, _PATH, Options.WRITE_FILE, Options.NOW]
  ),
  bind(_PATH).toValue('.')
 ];

modules/benchpress/src/runner.js

@@ -18,7 +18,7 @@ import { SampleDescription } from './sample_description';
 import { WebDriverAdapter } from './web_driver_adapter';
 import { Reporter } from './reporter';
 import { Metric } from './metric';
-import { Options } from './sample_options';
+import { Options } from './common_options';
 
 /**
  * The Runner is the main entry point for executing a sample run.
@@ -56,6 +56,7 @@ export class Runner {
 }
 
 var _DEFAULT_BINDINGS = [
+ Options.DEFAULT_BINDINGS,
  Sampler.BINDINGS,
  ConsoleReporter.BINDINGS,
  RegressionSlopeValidator.BINDINGS,

modules/benchpress/src/sample_description.js

@@ -2,7 +2,7 @@ import { StringMapWrapper, ListWrapper, StringMap } from 'angular2/src/facade/co
 import { bind, OpaqueToken } from 'angular2/di';
 import { Validator } from './validator';
 import { Metric } from './metric';
-import { Options } from './sample_options';
+import { Options } from './common_options';
 
 /**
  * SampleDescription merges all available descriptions about a sample
@@ -47,7 +47,5 @@ var _BINDINGS = [
  Metric, Options.SAMPLE_ID, Options.FORCE_GC, Options.USER_AGENT,
  Validator, Options.DEFAULT_DESCRIPTION, Options.SAMPLE_DESCRIPTION
  ]
- ),
- bind(Options.DEFAULT_DESCRIPTION).toValue({}),
- bind(Options.SAMPLE_DESCRIPTION).toValue({})
+ )
 ];

modules/benchpress/src/sampler.js

@@ -9,7 +9,7 @@ import { Reporter } from './reporter';
 import { WebDriverExtension } from './web_driver_extension';
 import { WebDriverAdapter } from './web_driver_adapter';
 
-import { Options } from './sample_options';
+import { Options } from './common_options';
 import { MeasureValues} from './measure_values';
 
 /**
@@ -23,8 +23,6 @@ import { MeasureValues} from './measure_values';
 export class Sampler {
  // TODO(tbosch): use static values when our transpiler supports them
  static get BINDINGS() { return _BINDINGS; }
- // TODO(tbosch): use static values when our transpiler supports them
- static get TIME() { return _TIME; }
 
  _driver:WebDriverAdapter;
  _driverExtension:WebDriverExtension;
@@ -34,14 +32,14 @@ export class Sampler {
  _forceGc:boolean;
  _prepare:Function;
  _execute:Function;
- _time:Function;
+ _now:Function;
 
  constructor({
- driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time
+ driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now
  }:{
  driver: WebDriverAdapter,
  driverExtension: WebDriverExtension, metric: Metric, reporter: Reporter,
- validator: Validator, prepare: Function, execute: Function, time: Function
+ validator: Validator, prepare: Function, execute: Function, now: Function
  }={}) {
  this._driver = driver;
  this._driverExtension = driverExtension;
@@ -51,7 +49,7 @@ export class Sampler {
  this._forceGc = forceGc;
  this._prepare = prepare;
  this._execute = execute;
- this._time = time;
+ this._now = now;
  }
 
  sample():Promise<SampleState> {
@@ -96,7 +94,7 @@ export class Sampler {
  }
 
  _report(state:SampleState, metricValues:StringMap):Promise<SampleState> {
- var measureValues = new MeasureValues(state.completeSample.length, this._time(), metricValues);
+ var measureValues = new MeasureValues(state.completeSample.length, this._now(), metricValues);
  var completeSample = ListWrapper.concat(state.completeSample, [measureValues]);
  var validSample = this._validator.validate(completeSample);
  var resultPromise = this._reporter.reportMeasureValues(measureValues);
@@ -118,11 +116,9 @@ export class SampleState {
  }
 }
 
-var _TIME = new OpaqueToken('Sampler.time');
-
 var _BINDINGS = [
  bind(Sampler).toFactory(
- (driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, time) => new Sampler({
+ (driver, driverExtension, metric, reporter, validator, forceGc, prepare, execute, now) => new Sampler({
  driver: driver,
  driverExtension: driverExtension,
  reporter: reporter,
@@ -134,14 +130,11 @@ var _BINDINGS = [
  // special null object, which is expensive.
  prepare: prepare !== false ? prepare : null,
  execute: execute,
- time: time
+ now: now
  }),
  [
  WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator,
- Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME
+ Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, Options.NOW
  ]
- ),
- bind(Options.FORCE_GC).toValue(false),
- bind(Options.PREPARE).toValue(false),
- bind(_TIME).toValue( () => DateWrapper.now() )
+ )
 ];

modules/benchpress/src/validator/regression_slope_validator.js

@@ -65,5 +65,5 @@ var _BINDINGS = [
  [_SAMPLE_SIZE, _METRIC]
  ),
  bind(_SAMPLE_SIZE).toValue(10),
- bind(_METRIC).toValue('script')
+ bind(_METRIC).toValue('scriptTime')
 ];