Fix issue with line-mode speedup (all language).

Lua not affected. Python 3 unique in having a larger bailout.

Author: Neil Fraser

csharp/DiffMatchPatch.cs

@@ -631,15 +631,20 @@ private string diff_linesToCharsMunge(string text, List<string> lineArray,
  lineEnd = text.Length - 1;
  }
  line = text.JavaSubstring(lineStart, lineEnd + 1);
- lineStart = lineEnd + 1;
 
  if (lineHash.ContainsKey(line)) {
  chars.Append(((char)(int)lineHash[line]));
  } else {
+ if (lineArray.Count == 65535) {
+ // Bail out at 65535 because char 65536 == char 0.
+ line = text.Substring(lineStart);
+ lineEnd = text.Length;
+ }
  lineArray.Add(line);
  lineHash.Add(line, lineArray.Count - 1);
  chars.Append(((char)(lineArray.Count - 1)));
  }
+ lineStart = lineEnd + 1;
  }
  return chars.ToString();
  }

csharp/tests/DiffMatchPatchTest.cs

@@ -124,10 +124,10 @@ public void diff_linesToCharsTest() {
  tmpVector.Clear();
  StringBuilder lineList = new StringBuilder();
  StringBuilder charList = new StringBuilder();
- for (int x = 1; x < n + 1; x++) {
- tmpVector.Add(x + "\n");
- lineList.Append(x + "\n");
- charList.Append(Convert.ToChar(x));
+ for (int i = 1; i < n + 1; i++) {
+ tmpVector.Add(i + "\n");
+ lineList.Append(i + "\n");
+ charList.Append(Convert.ToChar(i));
  }
  assertEquals("Test initialization fail #1.", n, tmpVector.Count);
  string lines = lineList.ToString();
@@ -164,10 +164,10 @@ public void diff_charsToLinesTest() {
  tmpVector.Clear();
  StringBuilder lineList = new StringBuilder();
  StringBuilder charList = new StringBuilder();
- for (int x = 1; x < n + 1; x++) {
- tmpVector.Add(x + "\n");
- lineList.Append(x + "\n");
- charList.Append(Convert.ToChar (x));
+ for (int i = 1; i < n + 1; i++) {
+ tmpVector.Add(i + "\n");
+ lineList.Append(i + "\n");
+ charList.Append(Convert.ToChar (i));
  }
  assertEquals("Test initialization fail #3.", n, tmpVector.Count);
  string lines = lineList.ToString();
@@ -178,6 +178,17 @@ public void diff_charsToLinesTest() {
  this.diff_charsToLines(diffs, tmpVector);
  assertEquals("diff_charsToLines: More than 256.", new List<Diff>
  {new Diff(Operation.DELETE, lines)}, diffs);
+
+ // More than 65536 to verify any 16-bit limitation.
+ lineList = new StringBuilder();
+ for (int i = 0; i < 66000; i++) {
+ lineList.Append(i + "\n");
+ }
+ chars = lineList.ToString();
+ Object[] result = this.diff_linesToChars(chars, "");
+ diffs = new List<Diff> {new Diff(Operation.INSERT, (string)result[0])};
+ this.diff_charsToLines(diffs, (List<string>)result[2]);
+ assertEquals("diff_charsToLines: More than 65536.", chars, diffs[0].text);
  }
 
  public void diff_cleanupMergeTest() {
@@ -692,9 +703,9 @@ public void diff_mainTest() {
  string a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n";
  string b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n";
  // Increase the text lengths by 1024 times to ensure a timeout.
- for (int x = 0; x < 10; x++) {
- a = a + a;
- b = b + b;
+ for (int i = 0; i < 10; i++) {
+ a += a;
+ b += b;
  }
  DateTime startTime = DateTime.Now;
  this.diff_main(a, b);

dart/DMPClass.dart

@@ -480,15 +480,23 @@ class DiffMatchPatch {
  lineEnd = text.length - 1;
  }
  line = text.substring(lineStart, lineEnd + 1);
- lineStart = lineEnd + 1;
 
  if (lineHash.containsKey(line)) {
  chars.writeCharCode(lineHash[line]);
  } else {
+ if (lineArray.length == 65535) {
+ // Bail out at 65535 because
+ // final chars1 = new StringBuffer();
+ // chars1.writeCharCode(65536);
+ // chars1.toString().codeUnitAt(0) == 55296;
+ line = text.substring(lineStart);
+ lineEnd = text.length;
+ }
  lineArray.add(line);
  lineHash[line] = lineArray.length - 1;
  chars.writeCharCode(lineArray.length - 1);
  }
+ lineStart = lineEnd + 1;
  }
  return chars.toString();
  }

dart/tests/DiffMatchPatchTest.dart

@@ -185,9 +185,9 @@ void testDiffLinesToChars() {
  int n = 300;
  List<String> lineList = [];
  StringBuffer charList = new StringBuffer();
- for (int x = 1; x < n + 1; x++) {
- lineList.add('$x\n');
- charList.writeCharCode(x);
+ for (int i = 1; i < n + 1; i++) {
+ lineList.add('$i\n');
+ charList.writeCharCode(i);
  }
  Expect.equals(n, lineList.length, 'Test initialization fail #1.');
  String lines = lineList.join();
@@ -212,9 +212,9 @@ void testDiffCharsToLines() {
  int n = 300;
  List<String> lineList = [];
  StringBuffer charList = new StringBuffer();
- for (int x = 1; x < n + 1; x++) {
- lineList.add('$x\n');
- charList.writeCharCode(x);
+ for (int i = 1; i < n + 1; i++) {
+ lineList.add('$i\n');
+ charList.writeCharCode(i);
  }
  Expect.equals(n, lineList.length, 'Test initialization fail #3.');
  String lines = lineList.join();
@@ -224,6 +224,17 @@ void testDiffCharsToLines() {
  diffs = [new Diff(Operation.delete, chars)];
  dmp._diff_charsToLines(diffs, lineList);
  Expect.listEquals([new Diff(Operation.delete, lines)], diffs, 'diff_charsToLines: More than 256.');
+
+ // More than 65536 to verify any 16-bit limitation.
+ lineList = [];
+ for (int i = 0; i < 66000; i++) {
+ lineList.add('$i\n');
+ }
+ chars = lineList.join();
+ final results = dmp._diff_linesToChars(chars, '');
+ diffs = [new Diff(Operation.insert, results['chars1'])];
+ dmp._diff_charsToLines(diffs, results['lineArray']);
+ Expect.equals(chars, diffs[0].text, 'diff_charsToLines: More than 65536.');
 }
 
 void testDiffCleanupMerge() {
@@ -532,7 +543,7 @@ void testDiffMain() {
  String a = '`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n';
  String b = 'I am the very model of a modern major general,\nI\'ve information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n';
  // Increase the text lengths by 1024 times to ensure a timeout.
- for (int x = 0; x < 10; x++) {
+ for (int i = 0; i < 10; i++) {
  a += a;
  b += b;
  }

java/src/name/fraser/neil/plaintext/diff_match_patch.java

@@ -545,15 +545,21 @@ private String diff_linesToCharsMunge(String text, List<String> lineArray,
  lineEnd = text.length() - 1;
  }
  line = text.substring(lineStart, lineEnd + 1);
- lineStart = lineEnd + 1;
 
  if (lineHash.containsKey(line)) {
  chars.append(String.valueOf((char) (int) lineHash.get(line)));
  } else {
+ if (lineArray.size() == 65535) {
+ // Bail out at 65535 because
+ // String.valueOf((char) 65536).equals(String.valueOf(((char) 0)))
+ line = text.substring(lineStart);
+ lineEnd = text.length();
+ }
  lineArray.add(line);
  lineHash.put(line, lineArray.size() - 1);
  chars.append(String.valueOf((char) (lineArray.size() - 1)));
  }
+ lineStart = lineEnd + 1;
  }
  return chars.toString();
  }

java/tests/name/fraser/neil/plaintext/diff_match_patch_test.java

@@ -135,10 +135,10 @@ public static void testDiffLinesToChars() {
  tmpVector.clear();
  StringBuilder lineList = new StringBuilder();
  StringBuilder charList = new StringBuilder();
- for (int x = 1; x < n + 1; x++) {
- tmpVector.add(x + "\n");
- lineList.append(x + "\n");
- charList.append(String.valueOf((char) x));
+ for (int i = 1; i < n + 1; i++) {
+ tmpVector.add(i + "\n");
+ lineList.append(i + "\n");
+ charList.append(String.valueOf((char) i));
  }
  assertEquals("Test initialization fail #1.", n, tmpVector.size());
  String lines = lineList.toString();
@@ -168,10 +168,10 @@ public static void testDiffCharsToLines() {
  tmpVector.clear();
  StringBuilder lineList = new StringBuilder();
  StringBuilder charList = new StringBuilder();
- for (int x = 1; x < n + 1; x++) {
- tmpVector.add(x + "\n");
- lineList.append(x + "\n");
- charList.append(String.valueOf((char) x));
+ for (int i = 1; i < n + 1; i++) {
+ tmpVector.add(i + "\n");
+ lineList.append(i + "\n");
+ charList.append(String.valueOf((char) i));
  }
  assertEquals("Test initialization fail #3.", n, tmpVector.size());
  String lines = lineList.toString();
@@ -181,6 +181,17 @@ public static void testDiffCharsToLines() {
  diffs = diffList(new Diff(DELETE, chars));
  dmp.diff_charsToLines(diffs, tmpVector);
  assertEquals("diff_charsToLines: More than 256.", diffList(new Diff(DELETE, lines)), diffs);
+
+ // More than 65536 to verify any 16-bit limitation.
+ lineList = new StringBuilder();
+ for (int i = 0; i < 66000; i++) {
+ lineList.append(i + "\n");
+ }
+ chars = lineList.toString();
+ LinesToCharsResult results = dmp.diff_linesToChars(chars, "");
+ diffs = diffList(new Diff(INSERT, results.chars1));
+ dmp.diff_charsToLines(diffs, results.lineArray);
+ assertEquals("diff_charsToLines: More than 65536.", chars, diffs.getFirst().text);
  }
 
  public static void testDiffCleanupMerge() {
@@ -500,9 +511,9 @@ public static void testDiffMain() {
  String a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n";
  String b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n";
  // Increase the text lengths by 1024 times to ensure a timeout.
- for (int x = 0; x < 10; x++) {
- a = a + a;
- b = b + b;
+ for (int i = 0; i < 10; i++) {
+ a += a;
+ b += b;
  }
  long startTime = System.currentTimeMillis();
  dmp.diff_main(a, b);

javascript/tests/diff_match_patch_test.js

@@ -570,7 +570,7 @@ function testDiffMain() {
  var a = '`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n';
  var b = 'I am the very model of a modern major general,\nI\'ve information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n';
  // Increase the text lengths by 1024 times to ensure a timeout.
- for (var x = 0; x < 10; x++) {
+ for (var i = 0; i < 10; i++) {
  a += a;
  b += b;
  }
@@ -603,7 +603,7 @@ function testDiffMain() {
  var texts_linemode = diff_rebuildtexts(dmp.diff_main(a, b, true));
  var texts_textmode = diff_rebuildtexts(dmp.diff_main(a, b, false));
  assertEquivalent(texts_textmode, texts_linemode);
- 
+
  // Test null inputs.
  try {
  dmp.diff_main(null, null);

objectivec/DiffMatchPatchCFUtilities.c

@@ -480,14 +480,18 @@ CFStringRef diff_linesToCharsMungeCFStringCreate(CFStringRef text, CFMutableArra
  }*/
 
  line = diff_CFStringCreateJavaSubstring(text, lineStart, lineEnd + 1);
- lineStart = lineEnd + 1;
 
  if (CFDictionaryContainsKey(lineHash, line)) {
  CFDictionaryGetValueIfPresent(lineHash, line, (const void **)&hashNumber);
  CFNumberGetValue(hashNumber, kCFNumberCFIndexType, &hash);
  const UniChar hashChar = (UniChar)hash;
  CFStringAppendCharacters(chars, &hashChar, 1);
  } else {
+ if (CFArrayGetCount(lineArray) == 65535) {
+ // Bail out at 65535 because char 65536 == char 0.
+ line = diff_CFStringCreateJavaSubstring(text, lineStart, textLength);
+ lineEnd = textLength;
+ }
  CFArrayAppendValue(lineArray, line);
  hash = CFArrayGetCount(lineArray) - 1;
  hashNumber = CFNumberCreate(kCFAllocatorDefault, kCFNumberCFIndexType, &hash);
@@ -497,6 +501,7 @@ CFStringRef diff_linesToCharsMungeCFStringCreate(CFStringRef text, CFMutableArra
  const UniChar hashChar = (UniChar)hash;
  CFStringAppendCharacters(chars, &hashChar, 1);
  }
+ lineStart = lineEnd + 1;
 
  CFRelease(line);
  }

objectivec/Tests/DiffMatchPatchTest.m

@@ -172,11 +172,11 @@ - (void)test_diff_linesToCharsTest {
  NSMutableString *lines = [NSMutableString string];
  NSMutableString *chars = [NSMutableString string];
  NSString *currentLine;
- for (unichar x = 1; x < n + 1; x++) {
- currentLine = [NSString stringWithFormat:@"%d\n", (int)x];
+ for (unichar i = 1; i < n + 1; i++) {
+ currentLine = [NSString stringWithFormat:@"%d\n", (int)i];
  [tmpVector addObject:currentLine];
  [lines appendString:currentLine];
- [chars appendString:[NSString stringWithFormat:@"%C", x]];
+ [chars appendString:[NSString stringWithFormat:@"%C", i]];
  }
  XCTAssertEqual((NSUInteger)n, tmpVector.count, @"More than 256 #1.");
  XCTAssertEqual((NSUInteger)n, chars.length, @"More than 256 #2.");
@@ -212,11 +212,11 @@ - (void)test_diff_charsToLinesTest {
  NSMutableString *lines = [NSMutableString string];
  NSMutableString *chars = [NSMutableString string];
  NSString *currentLine;
- for (unichar x = 1; x < n + 1; x++) {
- currentLine = [NSString stringWithFormat:@"%d\n", (int)x];
+ for (unichar i = 1; i < n + 1; i++) {
+ currentLine = [NSString stringWithFormat:@"%d\n", (int)i];
  [tmpVector addObject:currentLine];
  [lines appendString:currentLine];
- [chars appendString:[NSString stringWithFormat:@"%C", x]];
+ [chars appendString:[NSString stringWithFormat:@"%C", i]];
  }
  XCTAssertEqual((NSUInteger)n, tmpVector.count, @"More than 256 #1.");
  XCTAssertEqual((NSUInteger)n, chars.length, @"More than 256 #2.");
@@ -225,6 +225,19 @@ - (void)test_diff_charsToLinesTest {
  [dmp diff_chars:diffs toLines:tmpVector];
  XCTAssertEqualObjects([NSArray arrayWithObject:[Diff diffWithOperation:DIFF_DELETE andText:lines]], diffs, @"More than 256 #3.");
 
+ // More than 65536 to verify any 16-bit limitation.
+ lines = [NSMutableString string];
+ for (int i = 1; i < 66000; i++) {
+ currentLine = [NSString stringWithFormat:@"%d\n", i];
+ [lines appendString:currentLine];
+ }
+ NSArray *result;
+ result = [dmp diff_linesToCharsForFirstString:lines andSecondString:@""];
+ diffs = [NSArray arrayWithObject:[Diff diffWithOperation:DIFF_INSERT andText:result[0]]];
+ [dmp diff_chars:diffs toLines:result[2]];
+ Diff *myDiff = diffs.firstObject;
+ XCTAssertEqualObjects(lines, myDiff.text, @"More than 65536.");
+
  [dmp release];
 }
 
@@ -859,7 +872,7 @@ - (void)test_diff_mainTest {
  NSMutableString *aMutable = [NSMutableString stringWithString:a];
  NSMutableString *bMutable = [NSMutableString stringWithString:b];
  // Increase the text lengths by 1024 times to ensure a timeout.
- for (int x = 0; x < 10; x++) {
+ for (int i = 0; i < 10; i++) {
  [aMutable appendString:aMutable];
  [bMutable appendString:bMutable];
  }

python2/diff_match_patch.py

@@ -422,14 +422,18 @@ def diff_linesToCharsMunge(text):
  if lineEnd == -1:
  lineEnd = len(text) - 1
  line = text[lineStart:lineEnd + 1]
- lineStart = lineEnd + 1
 
  if line in lineHash:
  chars.append(unichr(lineHash[line]))
  else:
+ if len(lineArray) == 65535:
+ # Bail out at 65535 because unichr(65536) throws.
+ line = text[lineStart:]
+ lineEnd = len(text)
  lineArray.append(line)
  lineHash[line] = len(lineArray) - 1
  chars.append(unichr(len(lineArray) - 1))
+ lineStart = lineEnd + 1
  return "".join(chars)
 
  chars1 = diff_linesToCharsMunge(text1)