feat(broccoli): improve merge-trees plugin and add "overwrite" option

Author: caitp

tools/broccoli/broccoli-merge-trees.spec.ts

@@ -21,15 +21,15 @@ describe('MergeTrees', () => {
 
  function read(path) { return fs.readFileSync(path, "utf-8"); }
 
- it('should copy the file from the right-most inputTree', () => {
+ it('should copy the file from the right-most inputTree with overwrite=true', () => {
  let testDir: any = {
  'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
  'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
  'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})}
  };
  mockfs(testDir);
  let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
- let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {});
+ let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {overwrite: true});
  treeMerger.rebuild(treeDiffer.diffTrees());
  expect(read('dest/foo.js')).toBe('tree3/foo.js content');
 
@@ -44,4 +44,25 @@ describe('MergeTrees', () => {
  treeMerger.rebuild(treeDiffer.diffTrees());
  expect(read('dest/foo.js')).toBe('tree2/foo.js content');
  });
+
+ it('should throw if duplicates are used by default', () => {
+ let testDir: any = {
+ 'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
+ 'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
+ 'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})}
+ };
+ mockfs(testDir);
+ let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
+ let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {});
+ expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+
+ delete testDir.tree2['foo.js'];
+ delete testDir.tree3['foo.js'];
+ mockfs(testDir);
+ expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).not.toThrow();
+
+ testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)});
+ mockfs(testDir);
+ expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+ });
 });

tools/broccoli/broccoli-merge-trees.ts

@@ -4,81 +4,114 @@ import path = require('path');
 var symlinkOrCopySync = require('symlink-or-copy').sync;
 import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
 
-function pathExists(filePath) {
- try {
- if (fs.statSync(filePath)) {
- return true;
- }
- } catch (e) {
- if (e.code !== "ENOENT") {
- throw e;
- }
- }
- return false;
+interface MergeTreesOptions {
+ overwrite?: boolean;
 }
 
 function outputFileSync(sourcePath, destPath) {
  let dirname = path.dirname(destPath);
  fse.mkdirsSync(dirname, {fs: fs});
- fse.removeSync(destPath);
  symlinkOrCopySync(sourcePath, destPath);
 }
 
 export class MergeTrees implements DiffingBroccoliPlugin {
- private mergedPaths: {[key: string]: number} = Object.create(null);
+ private pathCache: {[key: string]: number[]} = Object.create(null);
+ public options: MergeTreesOptions;
+ private firstBuild: boolean = true;
 
- constructor(public inputPaths: string[], public cachePath: string, public options) {}
+ constructor(public inputPaths: string[], public cachePath: string,
+ options: MergeTreesOptions = {}) {
+ this.options = options || {};
+ }
 
  rebuild(treeDiffs: DiffResult[]) {
- treeDiffs.forEach((treeDiff: DiffResult, index) => {
- let inputPath = this.inputPaths[index];
- let existsLater = (relativePath) => {
- for (let i = treeDiffs.length - 1; i > index; --i) {
- if (pathExists(path.join(this.inputPaths[i], relativePath))) {
- return true;
+ let overwrite = this.options.overwrite;
+ let pathsToEmit: string[] = [];
+ let pathsToRemove: string[] = [];
+ let emitted: {[key: string]: boolean} = Object.create(null);
+ let contains = (cache, val) => {
+ for (let i = 0, ii = cache.length; i < ii; ++i) {
+ if (cache[i] === val) return true;
+ }
+ return false;
+ };
+
+ let emit = (relativePath) => {
+ // ASSERT(!emitted[relativePath]);
+ pathsToEmit.push(relativePath);
+ emitted[relativePath] = true;
+ };
+
+ if (this.firstBuild) {
+ // Build initial cache
+ treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+ index = treeDiffs.length - 1 - index;
+ treeDiff.changedPaths.forEach((changedPath) => {
+ let cache = this.pathCache[changedPath];
+ if (cache === undefined) {
+ this.pathCache[changedPath] = [index];
+ pathsToEmit.push(changedPath);
+ } else if (overwrite) {
+ // ASSERT(contains(pathsToEmit, changedPath));
+ cache.unshift(index);
+ } else {
+ throw new Error("`overwrite` option is required for handling duplicates.");
+ }
+ });
+ });
+ this.firstBuild = false;
+ } else {
+ // Update cache
+ treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+ index = treeDiffs.length - 1 - index;
+ treeDiff.removedPaths.forEach((removedPath) => {
+ let cache = this.pathCache[removedPath];
+ // ASSERT(cache !== undefined);
+ // ASSERT(contains(cache, index));
+ if (cache[cache.length - 1] === index) {
+ pathsToRemove.push(path.join(this.cachePath, removedPath));
+ cache.pop();
+ if (cache.length === 0) {
+ this.pathCache[removedPath] = undefined;
+ } else if (!emitted[removedPath]) {
+ if (cache.length === 1 && !overwrite) {
+ throw new Error("`overwrite` option is required for handling duplicates.");
+ }
+ emit(removedPath);
+ }
  }
- }
- return false;
- };
- let existsSooner = (relativePath) => {
- for (let i = index - 1; i >= 0; --i) {
- if (pathExists(path.join(this.inputPaths[i], relativePath))) {
- return i;
+ });
+ treeDiff.changedPaths.forEach((changedPath) => {
+ let cache = this.pathCache[changedPath];
+ if (cache === undefined) {
+ // File was added
+ this.pathCache[changedPath] = [index];
+ emit(changedPath);
+ } else if (!contains(cache, index)) {
+ cache.push(index);
+ cache.sort((a, b) => a - b);
+ if (cache.length > 1 && !overwrite) {
+ throw new Error("`overwrite` option is required for handling duplicates.");
+ }
+ if (cache[cache.length - 1] === index && !emitted[changedPath]) {
+ emit(changedPath);
+ }
  }
- }
- return -1;
- };
- treeDiff.changedPaths.forEach((changedPath) => {
- let inputTreeIndex = this.mergedPaths[changedPath];
- if (inputTreeIndex !== index && !existsLater(changedPath)) {
- inputTreeIndex = this.mergedPaths[changedPath] = index;
- let sourcePath = path.join(inputPath, changedPath);
- let destPath = path.join(this.cachePath, changedPath);
- outputFileSync(sourcePath, destPath);
- }
+ });
  });
+ }
 
-  treeDiff.removedPaths.forEach((removedPath) => {
-  let inputTreeIndex = this.mergedPaths[removedPath];
-
-  // if inputTreeIndex !== index, this same file was handled during
-  // changedPaths handling
-  if (inputTreeIndex !== index) return;
-
-  let destPath = path.join(this.cachePath, removedPath);
+ pathsToRemove.forEach((destPath) => fse.removeSync(destPath));
+ pathsToEmit.forEach((emittedPath) => {
+ let cache = this.pathCache[emittedPath];
+ let destPath = path.join(this.cachePath, emittedPath);
+ let sourceIndex = cache[cache.length - 1];
+ let sourceInputPath = this.inputPaths[sourceIndex];
+ let sourcePath = path.join(sourceInputPath, emittedPath);
+ if (cache.length > 1) {
  fse.removeSync(destPath);
- let newInputTreeIndex = existsSooner(removedPath);
-
- // Update cached value (to either newInputTreeIndex value or undefined)
- this.mergedPaths[removedPath] = newInputTreeIndex;
-
- if (newInputTreeIndex >= 0) {
- // Copy the file from the newInputTreeIndex inputPath if necessary.
- let newInputPath = this.inputPaths[newInputTreeIndex];
- let sourcePath = path.join(newInputPath, removedPath);
- outputFileSync(sourcePath, destPath);
- }
- });
+ }
+ outputFileSync(sourcePath, destPath);
  });
  }
 }