Support passing a start/length when creating a scanner.

Conflicts:	src/compiler/parser.ts	src/compiler/scanner.ts

Author: CyrusNajmabadi

src/compiler/parser.ts

@@ -5416,4 +5416,4 @@ module ts {
  Value = -1
  }
  }
-}
+}

src/compiler/scanner.ts

@@ -24,7 +24,9 @@ module ts {
  reScanSlashToken(): SyntaxKind;
  reScanTemplateToken(): SyntaxKind;
  scan(): SyntaxKind;
- setText(text: string): void;
+ // Sets the text for the scanner to scan. An optional subrange starting point and length
+ // can be provided to have the scanner only scan a portion of the text.
+ setText(text: string, start?: number, length?: number): void;
  setOnError(onError: ErrorCallback): void;
  setScriptTarget(scriptTarget: ScriptTarget): void;
  setTextPos(textPos: number): void;
@@ -597,11 +599,11 @@ module ts {
  ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);
  }
 
- /* @internal */ 
- export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {
+ // Creates a scanner over a (possibly unspecified) range of a piece of text.
+ /* @internal */ 
+ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, start?: number, length?: number): Scanner {
  let pos: number; // Current position (end position of text of current token)
- let len: number; // Length of text
-
+ let end: number; // end of text
  let startPos: number; // Start position of whitespace before current token
  let tokenPos: number; // Start position of text of current token
  let token: SyntaxKind;
@@ -610,7 +612,7 @@ module ts {
  let hasExtendedUnicodeEscape: boolean;
  let tokenIsUnterminated: boolean;
 
- setText(text);
+ setText(text, start, length);
 
  return {
  getStartPos: () => startPos,
@@ -732,7 +734,7 @@ module ts {
  let result = "";
  let start = pos;
  while (true) {
- if (pos >= len) {
+ if (pos >= end) {
  result += text.substring(start, pos);
  tokenIsUnterminated = true;
  error(Diagnostics.Unterminated_string_literal);
@@ -774,7 +776,7 @@ module ts {
  let resultingToken: SyntaxKind;
 
  while (true) {
- if (pos >= len) {
+ if (pos >= end) {
  contents += text.substring(start, pos);
  tokenIsUnterminated = true;
  error(Diagnostics.Unterminated_template_literal);
@@ -793,7 +795,7 @@ module ts {
  }
 
  // '${'
- if (currChar === CharacterCodes.$ && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) {
+ if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) {
  contents += text.substring(start, pos);
  pos += 2;
  resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle;
@@ -814,7 +816,7 @@ module ts {
  contents += text.substring(start, pos);
  pos++;
 
- if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
+ if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
  pos++;
  }
 
@@ -834,7 +836,7 @@ module ts {
 
  function scanEscapeSequence(): string {
  pos++;
- if (pos >= len) {
+ if (pos >= end) {
  error(Diagnostics.Unexpected_end_of_text);
  return "";
  }
@@ -860,7 +862,7 @@ module ts {
  return "\"";
  case CharacterCodes.u:
  // '\u{DDDDDDDD}'
- if (pos < len && text.charCodeAt(pos) === CharacterCodes.openBrace) {
+ if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
  hasExtendedUnicodeEscape = true;
  pos++;
  return scanExtendedUnicodeEscape();
@@ -876,7 +878,7 @@ module ts {
  // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence),
  // the line terminator is interpreted to be "the empty code unit sequence".
  case CharacterCodes.carriageReturn:
- if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
+ if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
  pos++;
  }
  // fall through
@@ -915,7 +917,7 @@ module ts {
  isInvalidExtendedEscape = true;
  }
 
- if (pos >= len) {
+ if (pos >= end) {
  error(Diagnostics.Unexpected_end_of_text);
  isInvalidExtendedEscape = true;
  }
@@ -952,7 +954,7 @@ module ts {
  // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX'
  // and return code point value if valid Unicode escape is found. Otherwise return -1.
  function peekUnicodeEscape(): number {
- if (pos + 5 < len && text.charCodeAt(pos + 1) === CharacterCodes.u) {
+ if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) {
  let start = pos;
  pos += 2;
  let value = scanExactNumberOfHexDigits(4);
@@ -965,7 +967,7 @@ module ts {
  function scanIdentifierParts(): string {
  let result = "";
  let start = pos;
- while (pos < len) {
+ while (pos < end) {
  let ch = text.charCodeAt(pos);
  if (isIdentifierPart(ch)) {
  pos++;
@@ -1032,7 +1034,7 @@ module ts {
  tokenIsUnterminated = false;
  while (true) {
  tokenPos = pos;
- if (pos >= len) {
+ if (pos >= end) {
  return token = SyntaxKind.EndOfFileToken;
  }
  let ch = text.charCodeAt(pos);
@@ -1045,7 +1047,7 @@ module ts {
  continue;
  }
  else {
- if (ch === CharacterCodes.carriageReturn && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) {
+ if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) {
  // consume both CR and LF
  pos += 2;
  }
@@ -1063,7 +1065,7 @@ module ts {
  continue;
  }
  else {
- while (pos < len && isWhiteSpace(text.charCodeAt(pos))) {
+ while (pos < end && isWhiteSpace(text.charCodeAt(pos))) {
  pos++;
  }
  return token = SyntaxKind.WhitespaceTrivia;
@@ -1136,7 +1138,7 @@ module ts {
  if (text.charCodeAt(pos + 1) === CharacterCodes.slash) {
  pos += 2;
 
- while (pos < len) {
+ while (pos < end) {
  if (isLineBreak(text.charCodeAt(pos))) {
  break;
  }
@@ -1156,7 +1158,7 @@ module ts {
  pos += 2;
 
  let commentClosed = false;
- while (pos < len) {
+ while (pos < end) {
  let ch = text.charCodeAt(pos);
 
  if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) {
@@ -1191,7 +1193,7 @@ module ts {
  return pos++, token = SyntaxKind.SlashToken;
 
  case CharacterCodes._0:
- if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
+ if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
  pos += 2;
  let value = scanMinimumNumberOfHexDigits(1);
  if (value < 0) {
@@ -1201,7 +1203,7 @@ module ts {
  tokenValue = "" + value;
  return token = SyntaxKind.NumericLiteral;
  }
- else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
+ else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
  pos += 2;
  let value = scanBinaryOrOctalDigits(/* base */ 2);
  if (value < 0) {
@@ -1211,7 +1213,7 @@ module ts {
  tokenValue = "" + value;
  return token = SyntaxKind.NumericLiteral;
  }
- else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
+ else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
  pos += 2;
  let value = scanBinaryOrOctalDigits(/* base */ 8);
  if (value < 0) {
@@ -1222,7 +1224,7 @@ module ts {
  return token = SyntaxKind.NumericLiteral;
  }
  // Try to parse as an octal
- if (pos + 1 < len && isOctalDigit(text.charCodeAt(pos + 1))) {
+ if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
  tokenValue = "" + scanOctalDigits();
  return token = SyntaxKind.NumericLiteral;
  }
@@ -1337,7 +1339,7 @@ module ts {
  default:
  if (isIdentifierStart(ch)) {
  pos++;
- while (pos < len && isIdentifierPart(ch = text.charCodeAt(pos))) pos++;
+ while (pos < end && isIdentifierPart(ch = text.charCodeAt(pos))) pos++;
  tokenValue = text.substring(tokenPos, pos);
  if (ch === CharacterCodes.backslash) {
  tokenValue += scanIdentifierParts();
@@ -1388,7 +1390,7 @@ module ts {
  while (true) {
  // If we reach the end of a file, or hit a newline, then this is an unterminated
  // regex. Report error and return what we have so far.
- if (p >= len) {
+ if (p >= end) {
  tokenIsUnterminated = true;
  error(Diagnostics.Unterminated_regular_expression_literal)
  break;
@@ -1424,7 +1426,7 @@ module ts {
  p++;
  }
 
- while (p < len && isIdentifierPart(text.charCodeAt(p))) {
+ while (p < end && isIdentifierPart(text.charCodeAt(p))) {
  p++;
  }
  pos = p;
@@ -1473,10 +1475,10 @@ module ts {
  return speculationHelper(callback, /*isLookahead:*/ false);
  }
 
- function setText(newText: string) {
+ function setText(newText: string, start: number, length: number) {
  text = newText || "";
- len = text.length;
- setTextPos(0);
+ end = length === undefined ? text.length : start + length;
+ setTextPos(start || 0);
  }
 
  function setOnError(errorCallback: ErrorCallback) {
@@ -1488,6 +1490,7 @@ module ts {
  }
 
  function setTextPos(textPos: number) {
+ Debug.assert(textPos >= 0);
  pos = textPos;
  startPos = textPos;
  tokenPos = textPos;

src/compiler/utilities.ts

@@ -274,8 +274,7 @@ module ts {
  }
 
  export function getSpanOfTokenAtPosition(sourceFile: SourceFile, pos: number): TextSpan {
- let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text);
- scanner.setTextPos(pos);
+ let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text, /*onError:*/ undefined, pos);
  scanner.scan();
  let start = scanner.getTokenPos();
  return createTextSpanFromBounds(start, scanner.getTextPos());