Merge pull request microsoft#2734 from Microsoft/singletonScanner

Use a single scanner for all parsing tasks.

Author: CyrusNajmabadi

src/compiler/checker.ts

@@ -12953,6 +12953,11 @@ module ts {
  }
  }
 
+ function isEvalOrArgumentsIdentifier(node: Node): boolean {
+ return node.kind === SyntaxKind.Identifier &&
+ ((<Identifier>node).text === "eval" || (<Identifier>node).text === "arguments");
+ }
+
  function checkGrammarConstructorTypeParameters(node: ConstructorDeclaration) {
  if (node.typeParameters) {
  return grammarErrorAtPos(getSourceFileOfNode(node), node.typeParameters.pos, node.typeParameters.end - node.typeParameters.pos, Diagnostics.Type_parameters_cannot_appear_on_a_constructor_declaration);

src/compiler/parser.ts

@@ -54,6 +54,7 @@ module ts {
  }
  text = "";
  }
+
  return text !== undefined ? createSourceFile(fileName, text, languageVersion, setParentNodes) : undefined;
  }
 

src/compiler/program.ts

@@ -25,6 +25,8 @@ module ts {
  reScanTemplateToken(): SyntaxKind;
  scan(): SyntaxKind;
  setText(text: string): void;
+ setOnError(onError: ErrorCallback): void;
+ setScriptTarget(scriptTarget: ScriptTarget): void;
  setTextPos(textPos: number): void;
  // Invokes the provided callback then unconditionally restores the scanner to the state it 
  // was in immediately prior to invoking the callback. The result of invoking the callback
@@ -599,6 +601,7 @@ module ts {
  export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {
  let pos: number; // Current position (end position of text of current token)
  let len: number; // Length of text
+
  let startPos: number; // Start position of whitespace before current token
  let tokenPos: number; // Start position of text of current token
  let token: SyntaxKind;
@@ -607,6 +610,32 @@ module ts {
  let hasExtendedUnicodeEscape: boolean;
  let tokenIsUnterminated: boolean;
 
+ setText(text);
+
+ return {
+ getStartPos: () => startPos,
+ getTextPos: () => pos,
+ getToken: () => token,
+ getTokenPos: () => tokenPos,
+ getTokenText: () => text.substring(tokenPos, pos),
+ getTokenValue: () => tokenValue,
+ hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
+ hasPrecedingLineBreak: () => precedingLineBreak,
+ isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
+ isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
+ isUnterminated: () => tokenIsUnterminated,
+ reScanGreaterToken,
+ reScanSlashToken,
+ reScanTemplateToken,
+ scan,
+ setText,
+ setScriptTarget,
+ setOnError,
+ setTextPos,
+ tryScan,
+ lookAhead,
+ };
+
  function error(message: DiagnosticMessage, length?: number): void {
  if (onError) {
  onError(message, length || 0);
@@ -1450,37 +1479,24 @@ module ts {
  setTextPos(0);
  }
 
+ function setOnError(errorCallback: ErrorCallback) {
+ onError = errorCallback;
+ }
+
+ function setScriptTarget(scriptTarget: ScriptTarget) {
+ languageVersion = scriptTarget;
+ }
+
  function setTextPos(textPos: number) {
  pos = textPos;
  startPos = textPos;
  tokenPos = textPos;
  token = SyntaxKind.Unknown;
  precedingLineBreak = false;
- }
-
- setText(text);
-
 
- return {
- getStartPos: () => startPos,
- getTextPos: () => pos,
- getToken: () => token,
- getTokenPos: () => tokenPos,
- getTokenText: () => text.substring(tokenPos, pos),
- getTokenValue: () => tokenValue,
- hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
- hasPrecedingLineBreak: () => precedingLineBreak,
- isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
- isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
- isUnterminated: () => tokenIsUnterminated,
- reScanGreaterToken,
- reScanSlashToken,
- reScanTemplateToken,
- scan,
- setText,
- setTextPos,
- tryScan,
- lookAhead,
- };
+ tokenValue = undefined;
+ hasExtendedUnicodeEscape = false;
+ tokenIsUnterminated = false;
+ }
  }
 }

src/compiler/scanner.ts

@@ -1612,6 +1612,55 @@ module ts {
  }
  }
 
+ export function modifierToFlag(token: SyntaxKind): NodeFlags {
+ switch (token) {
+ case SyntaxKind.StaticKeyword: return NodeFlags.Static;
+ case SyntaxKind.PublicKeyword: return NodeFlags.Public;
+ case SyntaxKind.ProtectedKeyword: return NodeFlags.Protected;
+ case SyntaxKind.PrivateKeyword: return NodeFlags.Private;
+ case SyntaxKind.ExportKeyword: return NodeFlags.Export;
+ case SyntaxKind.DeclareKeyword: return NodeFlags.Ambient;
+ case SyntaxKind.ConstKeyword: return NodeFlags.Const;
+ case SyntaxKind.DefaultKeyword: return NodeFlags.Default;
+ }
+ return 0;
+ }
+
+ export function isLeftHandSideExpression(expr: Expression): boolean {
+ if (expr) {
+ switch (expr.kind) {
+ case SyntaxKind.PropertyAccessExpression:
+ case SyntaxKind.ElementAccessExpression:
+ case SyntaxKind.NewExpression:
+ case SyntaxKind.CallExpression:
+ case SyntaxKind.TaggedTemplateExpression:
+ case SyntaxKind.ArrayLiteralExpression:
+ case SyntaxKind.ParenthesizedExpression:
+ case SyntaxKind.ObjectLiteralExpression:
+ case SyntaxKind.ClassExpression:
+ case SyntaxKind.FunctionExpression:
+ case SyntaxKind.Identifier:
+ case SyntaxKind.RegularExpressionLiteral:
+ case SyntaxKind.NumericLiteral:
+ case SyntaxKind.StringLiteral:
+ case SyntaxKind.NoSubstitutionTemplateLiteral:
+ case SyntaxKind.TemplateExpression:
+ case SyntaxKind.FalseKeyword:
+ case SyntaxKind.NullKeyword:
+ case SyntaxKind.ThisKeyword:
+ case SyntaxKind.TrueKeyword:
+ case SyntaxKind.SuperKeyword:
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ export function isAssignmentOperator(token: SyntaxKind): boolean {
+ return token >= SyntaxKind.FirstAssignment && token <= SyntaxKind.LastAssignment;
+ }
+
  // Returns false if this heritage clause element's expression contains something unsupported
  // (i.e. not a name or dotted name).
  export function isSupportedHeritageClauseElement(node: HeritageClauseElement): boolean {

src/compiler/utilities.ts

@@ -28,9 +28,9 @@ export function compile(fileNames: string[], options: ts.CompilerOptions): void
  var program = ts.createProgram(fileNames, options);
 >program : ts.Program, Symbol(program, Decl(APISample_compile.ts, 14, 7))
 >ts.createProgram(fileNames, options) : ts.Program
->ts.createProgram : (rootNames: string[], options: ts.CompilerOptions, host?: ts.CompilerHost) => ts.Program, Symbol(ts.createProgram, Decl(typescript.d.ts, 1229, 113))
+>ts.createProgram : (rootNames: string[], options: ts.CompilerOptions, host?: ts.CompilerHost) => ts.Program, Symbol(ts.createProgram, Decl(typescript.d.ts, 1225, 113))
 >ts : typeof ts, Symbol(ts, Decl(APISample_compile.ts, 9, 20))
->createProgram : (rootNames: string[], options: ts.CompilerOptions, host?: ts.CompilerHost) => ts.Program, Symbol(ts.createProgram, Decl(typescript.d.ts, 1229, 113))
+>createProgram : (rootNames: string[], options: ts.CompilerOptions, host?: ts.CompilerHost) => ts.Program, Symbol(ts.createProgram, Decl(typescript.d.ts, 1225, 113))
 >fileNames : string[], Symbol(fileNames, Decl(APISample_compile.ts, 13, 24))
 >options : ts.CompilerOptions, Symbol(options, Decl(APISample_compile.ts, 13, 44))
 
@@ -46,9 +46,9 @@ export function compile(fileNames: string[], options: ts.CompilerOptions): void
 >ts.getPreEmitDiagnostics(program).concat(emitResult.diagnostics) : ts.Diagnostic[]
 >ts.getPreEmitDiagnostics(program).concat : { <U extends ts.Diagnostic[]>(...items: U[]): ts.Diagnostic[]; (...items: ts.Diagnostic[]): ts.Diagnostic[]; }, Symbol(Array.concat, Decl(lib.d.ts, 1025, 13), Decl(lib.d.ts, 1030, 46))
 >ts.getPreEmitDiagnostics(program) : ts.Diagnostic[]
->ts.getPreEmitDiagnostics : (program: ts.Program) => ts.Diagnostic[], Symbol(ts.getPreEmitDiagnostics, Decl(typescript.d.ts, 1227, 98))
+>ts.getPreEmitDiagnostics : (program: ts.Program) => ts.Diagnostic[], Symbol(ts.getPreEmitDiagnostics, Decl(typescript.d.ts, 1223, 98))
 >ts : typeof ts, Symbol(ts, Decl(APISample_compile.ts, 9, 20))
->getPreEmitDiagnostics : (program: ts.Program) => ts.Diagnostic[], Symbol(ts.getPreEmitDiagnostics, Decl(typescript.d.ts, 1227, 98))
+>getPreEmitDiagnostics : (program: ts.Program) => ts.Diagnostic[], Symbol(ts.getPreEmitDiagnostics, Decl(typescript.d.ts, 1223, 98))
 >program : ts.Program, Symbol(program, Decl(APISample_compile.ts, 14, 7))
 >concat : { <U extends ts.Diagnostic[]>(...items: U[]): ts.Diagnostic[]; (...items: ts.Diagnostic[]): ts.Diagnostic[]; }, Symbol(Array.concat, Decl(lib.d.ts, 1025, 13), Decl(lib.d.ts, 1030, 46))
 >emitResult.diagnostics : ts.Diagnostic[], Symbol(ts.EmitResult.diagnostics, Decl(typescript.d.ts, 820, 29))
@@ -67,21 +67,21 @@ export function compile(fileNames: string[], options: ts.CompilerOptions): void
 >line : number, Symbol(line, Decl(APISample_compile.ts, 20, 13))
 >character : number, Symbol(character, Decl(APISample_compile.ts, 20, 19))
 >diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start) : ts.LineAndCharacter
->diagnostic.file.getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1290, 26))
+>diagnostic.file.getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1286, 26))
 >diagnostic.file : ts.SourceFile, Symbol(ts.Diagnostic.file, Decl(typescript.d.ts, 1062, 26))
 >diagnostic : ts.Diagnostic, Symbol(diagnostic, Decl(APISample_compile.ts, 19, 27))
 >file : ts.SourceFile, Symbol(ts.Diagnostic.file, Decl(typescript.d.ts, 1062, 26))
->getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1290, 26))
+>getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1286, 26))
 >diagnostic.start : number, Symbol(ts.Diagnostic.start, Decl(typescript.d.ts, 1063, 25))
 >diagnostic : ts.Diagnostic, Symbol(diagnostic, Decl(APISample_compile.ts, 19, 27))
 >start : number, Symbol(ts.Diagnostic.start, Decl(typescript.d.ts, 1063, 25))
 
  var message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
 >message : string, Symbol(message, Decl(APISample_compile.ts, 21, 11))
 >ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n') : string
->ts.flattenDiagnosticMessageText : (messageText: string | ts.DiagnosticMessageChain, newLine: string) => string, Symbol(ts.flattenDiagnosticMessageText, Decl(typescript.d.ts, 1228, 67))
+>ts.flattenDiagnosticMessageText : (messageText: string | ts.DiagnosticMessageChain, newLine: string) => string, Symbol(ts.flattenDiagnosticMessageText, Decl(typescript.d.ts, 1224, 67))
 >ts : typeof ts, Symbol(ts, Decl(APISample_compile.ts, 9, 20))
->flattenDiagnosticMessageText : (messageText: string | ts.DiagnosticMessageChain, newLine: string) => string, Symbol(ts.flattenDiagnosticMessageText, Decl(typescript.d.ts, 1228, 67))
+>flattenDiagnosticMessageText : (messageText: string | ts.DiagnosticMessageChain, newLine: string) => string, Symbol(ts.flattenDiagnosticMessageText, Decl(typescript.d.ts, 1224, 67))
 >diagnostic.messageText : string | ts.DiagnosticMessageChain, Symbol(ts.Diagnostic.messageText, Decl(typescript.d.ts, 1065, 23))
 >diagnostic : ts.Diagnostic, Symbol(diagnostic, Decl(APISample_compile.ts, 19, 27))
 >messageText : string | ts.DiagnosticMessageChain, Symbol(ts.Diagnostic.messageText, Decl(typescript.d.ts, 1065, 23))

tests/baselines/reference/APISample_compile.types

@@ -22,7 +22,7 @@ export function delint(sourceFile: ts.SourceFile) {
 >delint : (sourceFile: ts.SourceFile) => void, Symbol(delint, Decl(APISample_linter.ts, 11, 33))
 >sourceFile : ts.SourceFile, Symbol(sourceFile, Decl(APISample_linter.ts, 13, 23))
 >ts : any, Symbol(ts, Decl(APISample_linter.ts, 11, 6))
->SourceFile : ts.SourceFile, Symbol(ts.SourceFile, Decl(typescript.d.ts, 740, 5), Decl(typescript.d.ts, 1289, 5))
+>SourceFile : ts.SourceFile, Symbol(ts.SourceFile, Decl(typescript.d.ts, 740, 5), Decl(typescript.d.ts, 1285, 5))
 
  delintNode(sourceFile);
 >delintNode(sourceFile) : void
@@ -33,7 +33,7 @@ export function delint(sourceFile: ts.SourceFile) {
 >delintNode : (node: ts.Node) => void, Symbol(delintNode, Decl(APISample_linter.ts, 14, 27))
 >node : ts.Node, Symbol(node, Decl(APISample_linter.ts, 16, 24))
 >ts : any, Symbol(ts, Decl(APISample_linter.ts, 11, 6))
->Node : ts.Node, Symbol(ts.Node, Decl(typescript.d.ts, 296, 5), Decl(typescript.d.ts, 1249, 32))
+>Node : ts.Node, Symbol(ts.Node, Decl(typescript.d.ts, 296, 5), Decl(typescript.d.ts, 1245, 32))
 
  switch (node.kind) {
 >node.kind : ts.SyntaxKind, Symbol(ts.Node.kind, Decl(typescript.d.ts, 297, 38))
@@ -230,20 +230,20 @@ export function delint(sourceFile: ts.SourceFile) {
 >report : (node: ts.Node, message: string) => void, Symbol(report, Decl(APISample_linter.ts, 48, 5))
 >node : ts.Node, Symbol(node, Decl(APISample_linter.ts, 50, 20))
 >ts : any, Symbol(ts, Decl(APISample_linter.ts, 11, 6))
->Node : ts.Node, Symbol(ts.Node, Decl(typescript.d.ts, 296, 5), Decl(typescript.d.ts, 1249, 32))
+>Node : ts.Node, Symbol(ts.Node, Decl(typescript.d.ts, 296, 5), Decl(typescript.d.ts, 1245, 32))
 >message : string, Symbol(message, Decl(APISample_linter.ts, 50, 34))
 
  let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart());
 >line : number, Symbol(line, Decl(APISample_linter.ts, 51, 13))
 >character : number, Symbol(character, Decl(APISample_linter.ts, 51, 19))
 >sourceFile.getLineAndCharacterOfPosition(node.getStart()) : ts.LineAndCharacter
->sourceFile.getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1290, 26))
+>sourceFile.getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1286, 26))
 >sourceFile : ts.SourceFile, Symbol(sourceFile, Decl(APISample_linter.ts, 13, 23))
->getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1290, 26))
+>getLineAndCharacterOfPosition : (pos: number) => ts.LineAndCharacter, Symbol(ts.SourceFile.getLineAndCharacterOfPosition, Decl(typescript.d.ts, 1286, 26))
 >node.getStart() : number
->node.getStart : (sourceFile?: ts.SourceFile) => number, Symbol(ts.Node.getStart, Decl(typescript.d.ts, 1254, 53))
+>node.getStart : (sourceFile?: ts.SourceFile) => number, Symbol(ts.Node.getStart, Decl(typescript.d.ts, 1250, 53))
 >node : ts.Node, Symbol(node, Decl(APISample_linter.ts, 50, 20))
->getStart : (sourceFile?: ts.SourceFile) => number, Symbol(ts.Node.getStart, Decl(typescript.d.ts, 1254, 53))
+>getStart : (sourceFile?: ts.SourceFile) => number, Symbol(ts.Node.getStart, Decl(typescript.d.ts, 1250, 53))
 
  console.log(`${sourceFile.fileName} (${line + 1},${character + 1}): ${message}`);
 >console.log(`${sourceFile.fileName} (${line + 1},${character + 1}): ${message}`) : any
@@ -286,9 +286,9 @@ fileNames.forEach(fileName => {
  let sourceFile = ts.createSourceFile(fileName, readFileSync(fileName).toString(), ts.ScriptTarget.ES6, /*setParentNodes */ true);
 >sourceFile : ts.SourceFile, Symbol(sourceFile, Decl(APISample_linter.ts, 59, 7))
 >ts.createSourceFile(fileName, readFileSync(fileName).toString(), ts.ScriptTarget.ES6, /*setParentNodes */ true) : ts.SourceFile
->ts.createSourceFile : (fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, setParentNodes?: boolean) => ts.SourceFile, Symbol(ts.createSourceFile, Decl(typescript.d.ts, 1218, 62))
+>ts.createSourceFile : (fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, setParentNodes?: boolean) => ts.SourceFile, Symbol(ts.createSourceFile, Decl(typescript.d.ts, 1215, 107))
 >ts : typeof ts, Symbol(ts, Decl(APISample_linter.ts, 11, 6))
->createSourceFile : (fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, setParentNodes?: boolean) => ts.SourceFile, Symbol(ts.createSourceFile, Decl(typescript.d.ts, 1218, 62))
+>createSourceFile : (fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, setParentNodes?: boolean) => ts.SourceFile, Symbol(ts.createSourceFile, Decl(typescript.d.ts, 1215, 107))
 >fileName : any, Symbol(fileName, Decl(APISample_linter.ts, 57, 18))
 >readFileSync(fileName).toString() : any
 >readFileSync(fileName).toString : any

tests/baselines/reference/APISample_linter.types

@@ -19,9 +19,9 @@ const source = "let x: string = 'string'";
 let result = ts.transpile(source, { module: ts.ModuleKind.CommonJS });
 >result : string, Symbol(result, Decl(APISample_transform.ts, 13, 3))
 >ts.transpile(source, { module: ts.ModuleKind.CommonJS }) : string
->ts.transpile : (input: string, compilerOptions?: ts.CompilerOptions, fileName?: string, diagnostics?: ts.Diagnostic[]) => string, Symbol(ts.transpile, Decl(typescript.d.ts, 1756, 5))
+>ts.transpile : (input: string, compilerOptions?: ts.CompilerOptions, fileName?: string, diagnostics?: ts.Diagnostic[]) => string, Symbol(ts.transpile, Decl(typescript.d.ts, 1752, 5))
 >ts : typeof ts, Symbol(ts, Decl(APISample_transform.ts, 9, 6))
->transpile : (input: string, compilerOptions?: ts.CompilerOptions, fileName?: string, diagnostics?: ts.Diagnostic[]) => string, Symbol(ts.transpile, Decl(typescript.d.ts, 1756, 5))
+>transpile : (input: string, compilerOptions?: ts.CompilerOptions, fileName?: string, diagnostics?: ts.Diagnostic[]) => string, Symbol(ts.transpile, Decl(typescript.d.ts, 1752, 5))
 >source : string, Symbol(source, Decl(APISample_transform.ts, 11, 5))
 >{ module: ts.ModuleKind.CommonJS } : { [x: string]: ts.ModuleKind; module: ts.ModuleKind; }
 >module : ts.ModuleKind, Symbol(module, Decl(APISample_transform.ts, 13, 35))