make it easier to run all integration tests from local data

Author: trxcllnt

js/bin/integration.js

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+var fs = require('fs');
+var glob = require('glob');
 var path = require('path');
 var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
 var child_process = require(`child_process`);
@@ -29,12 +31,14 @@ var optionList = [
  {
  type: String,
  name: 'arrow', alias: 'a',
- description: 'The Arrow file to read/write'
+ multiple: true, defaultValue: [],
+ description: 'The Arrow file[s] to read/write'
  },
  {
  type: String,
  name: 'json', alias: 'j',
- description: 'The JSON file to read/write'
+ multiple: true, defaultValue: [],
+ description: 'The JSON file[s] to read/write'
  }
 ];
 
@@ -66,20 +70,60 @@ function print_usage() {
  process.exit(1);
 }
 
-if (!argv.arrow || !argv.json || !argv.mode) {
+let jsonPaths = argv.json;
+let arrowPaths = argv.arrow;
+
+if (!argv.mode) {
+ return print_usage();
+}
+
+let mode = argv.mode.toUpperCase();
+if (mode === 'VALIDATE' && !jsonPaths.length) {
+ jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`));
+ if (!arrowPaths.length) {
+ [jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => {
+ const { name } = path.parse(jsonPath);
+ for (const source of ['cpp', 'java']) {
+ for (const format of ['file', 'stream']) {
+ const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+ if (fs.existsSync(arrowPath)) {
+ jsonPaths.push(jsonPath);
+ arrowPaths.push(arrowPath);
+ console.log('-j', jsonPath, '-a', arrowPath, '\\');
+ }
+ }
+ }
+ return [jsonPaths, arrowPaths];
+ }, [[], []]);
+ }
+} else if (!jsonPaths.length) {
  return print_usage();
 }
 
-switch (argv.mode.toUpperCase()) {
+switch (mode) {
  case 'VALIDATE':
+ const args = [`test`, `-i`].concat(argv._unknown || []);
+ jsonPaths.forEach((p, i) => {
+ args.push('-j', p, '-a', arrowPaths[i]);
+ });
  child_process.spawnSync(
- gulp,
- [`test`, `-i`].concat(process.argv.slice(2)),
+ gulp, args,
  {
  cwd: path.resolve(__dirname, '..'),
  stdio: ['ignore', 'inherit', 'inherit']
  }
  );
+ // for (let i = -1, n = jsonPaths.length; ++i < n;) {
+ // const jsonPath = jsonPaths[i];
+ // const arrowPath = arrowPaths[i];
+ // child_process.spawnSync(
+ // gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
+ // {
+ // cwd: path.resolve(__dirname, '..'),
+ // stdio: ['ignore', 'inherit', 'inherit']
+ // }
+ // );
+ // }
  break;
  default:
  print_usage();

js/gulp/argv.js

@@ -22,13 +22,11 @@ const argv = require(`command-line-args`)([
  { name: `target`, type: String, defaultValue: `` },
  { name: `module`, type: String, defaultValue: `` },
  { name: `coverage`, type: Boolean, defaultValue: false },
- { name: `json_file`, alias: `j`, type: String, defaultValue: null },
- { name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
  { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
  { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
  { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
- { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
- { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
+ { name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] },
+ { name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] },
 ], { partial: true });
 
 const { targets, modules } = argv;

js/gulp/test-task.js

@@ -48,11 +48,9 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function
  opts.env = { ...opts.env,
  TEST_TARGET: target,
  TEST_MODULE: format,
- JSON_PATH: argv.json_file,
- ARROW_PATH: argv.arrow_file,
  TEST_TS_SOURCE: !!argv.coverage,
- TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
- TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
+ JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
+ ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
  };
  return !debug ?
  child_process.spawn(jest, args, opts) :

js/package.json

@@ -62,8 +62,8 @@
  "devDependencies": {
  "@std/esm": "0.19.7",
  "@types/flatbuffers": "1.6.5",
- "@types/glob": "5.0.34",
- "@types/jest": "22.0.1",
+ "@types/glob": "5.0.35",
+ "@types/jest": "22.1.0",
  "@types/node": "9.3.0",
  "ast-types": "0.10.1",
  "benchmark": "2.1.4",
@@ -78,9 +78,9 @@
  "gulp-rename": "1.2.2",
  "gulp-sourcemaps": "2.6.3",
  "gulp-transform-js-ast": "1.0.2",
- "gulp-typescript": "3.2.3",
+ "gulp-typescript": "3.2.4",
  "ix": "2.3.4",
- "jest": "22.1.3",
+ "jest": "22.1.4",
  "jest-environment-node-debug": "2.0.0",
  "json": "9.0.6",
  "lerna": "2.7.1",

js/test/integration/validate-tests.ts

@@ -18,28 +18,41 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-if (!process.env.JSON_PATH || !process.env.ARROW_PATH) {
- throw new Error('Integration tests need paths to both json and arrow files');
-}
+import Arrow from '../Arrow';
+import { zip } from 'ix/iterable/zip';
+import { toArray } from 'ix/iterable/toarray';
 
-const jsonPath = path.resolve(process.env.JSON_PATH + '');
-const arrowPath = path.resolve(process.env.ARROW_PATH + '');
+/* tslint:disable */
+const { parse: bignumJSONParse } = require('json-bignum');
 
-if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) {
- throw new Error('Integration tests need both json and arrow files to exist');
-}
+const { Table, read } = Arrow;
 
-/* tslint:disable */
-const { parse } = require('json-bignum');
+if (!process.env.JSON_PATHS || !process.env.ARROW_PATHS) {
+ throw new Error('Integration tests need paths to both json and arrow files');
+}
 
-const jsonData = parse(fs.readFileSync(jsonPath, 'utf8'));
-const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
+function resolvePathArgs(paths: string) {
+ let pathsArray = JSON.parse(paths) as string | string[];
+ return (Array.isArray(pathsArray) ? pathsArray : [pathsArray])
+ .map((p) => path.resolve(p))
+ .map((p) => {
+ if (fs.existsSync(p)) {
+ return p;
+ }
+ console.warn(`Could not find file "${p}"`);
+ return undefined;
+ });
+}
 
-import Arrow from '../Arrow';
-import { zip } from 'ix/iterable/zip';
-import { toArray } from 'ix/iterable/toarray';
+const getOrReadFileBuffer = ((cache: any) => function getFileBuffer(path: string, ...args: any[]) {
+ return cache[path] || (cache[path] = fs.readFileSync(path, ...args));
+})({});
 
-const { Table, read } = Arrow;
+const jsonAndArrowPaths = toArray(zip(
+ resolvePathArgs(process.env.JSON_PATHS!),
+ resolvePathArgs(process.env.ARROW_PATHS!)
+))
+.filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];
 
 expect.extend({
  toEqualVector(v1: any, v2: any) {
@@ -66,7 +79,7 @@ expect.extend({
 
  for (let i = -1, n = props.length; ++i < n;) {
  const prop = props[i];
- if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) {
+ if (`${v1[prop]}` !== `${v2[prop]}`) {
  propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`);
  }
  }
@@ -98,35 +111,43 @@ expect.extend({
 });
 
 describe(`Integration`, () => {
- testReaderIntegration();
- testTableFromBuffersIntegration();
+ for (const [jsonFilePath, arrowFilePath] of jsonAndArrowPaths) {
+ let { name, dir } = path.parse(arrowFilePath);
+ dir = dir.split(path.sep).slice(-2).join(path.sep);
+ const json = bignumJSONParse(getOrReadFileBuffer(jsonFilePath, 'utf8'));
+ const arrowBuffer = getOrReadFileBuffer(arrowFilePath) as Uint8Array;
+ describe(path.join(dir, name), () => {
+ testReaderIntegration(json, arrowBuffer);
+ testTableFromBuffersIntegration(json, arrowBuffer);
+ });
+ }
 });
 
-function testReaderIntegration() {
- test(`json and arrow buffers report the same values`, () => {
- debugger;
+function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+ test(`json and arrow record batches report the same values`, () => {
  expect.hasAssertions();
  const jsonRecordBatches = toArray(read(jsonData));
- const binaryRecordBatches = toArray(read(arrowBuffers));
+ const binaryRecordBatches = toArray(read(arrowBuffer));
  for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonRecordBatches, binaryRecordBatches)) {
  expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
  expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
  for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
+ (jsonRecordBatch.columns[i] as any).name = jsonRecordBatch.schema.fields[i].name;
  (expect(jsonRecordBatch.columns[i]) as any).toEqualVector(binaryRecordBatch.columns[i]);
  }
  }
  });
 }
 
-function testTableFromBuffersIntegration() {
- test(`json and arrow buffers report the same values`, () => {
- debugger;
+function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+ test(`json and arrow tables report the same values`, () => {
  expect.hasAssertions();
  const jsonTable = Table.from(jsonData);
- const binaryTable = Table.from(arrowBuffers);
+ const binaryTable = Table.from(arrowBuffer);
  expect(jsonTable.length).toEqual(binaryTable.length);
  expect(jsonTable.numCols).toEqual(binaryTable.numCols);
  for (let i = -1, n = jsonTable.numCols; ++i < n;) {
+ (jsonTable.columns[i] as any).name = jsonTable.schema.fields[i].name;
  (expect(jsonTable.columns[i]) as any).toEqualVector(binaryTable.columns[i]);
  }
  });