refactor(lexer): rename to scanner, use ints, etc.

Author: chirayuk

modules/change_detection/src/parser/lexer.js renamed to modules/change_detection/src/parser/scanner.js

@@ -1,5 +1,5 @@
 import {List, ListWrapper, SetWrapper} from "facade/collection";
-import {FIELD, NumberWrapper, StringJoiner, StringWrapper} from "facade/lang";
+import {int, FIELD, NumberWrapper, StringJoiner, StringWrapper} from "facade/lang";
 
 // TODO(chirayu): Rewrite as consts when possible.
 export var TOKEN_TYPE_CHARACTER = 1;
@@ -12,21 +12,21 @@ export var TOKEN_TYPE_NUMBER = 6;
 export class Token {
  @FIELD('final index:int')
  @FIELD('final type:int')
- @FIELD('final _intValue:int')
+ @FIELD('final _numValue:int')
  @FIELD('final _strValue:int')
- constructor(index:number/*int*/, type:number/*int*/, intValue:number/*int*/, strValue:string) {
+ constructor(index:int, type:int, numValue:number, strValue:string) {
  /**
  * NOTE: To ensure that this constructor creates the same hidden class each time, ensure that
  * all the fields are assigned to in the exact same order in each run of this constructor.
  */
  this.index = index;
  this.type = type;
- this._intValue = intValue;
+ this._numValue = numValue;
  this._strValue = strValue;
  }
 
- isCharacter(code:number/*int*/):boolean {
- return (this.type == TOKEN_TYPE_CHARACTER && this._intValue == code);
+ isCharacter(code:int):boolean {
+ return (this.type == TOKEN_TYPE_CHARACTER && this._numValue == code);
  }
 
  isNumber():boolean {
@@ -65,44 +65,44 @@ export class Token {
  return (this.type == TOKEN_TYPE_KEYWORD && this._strValue == "false");
  }
 
- toNumber():number/*int*/ {
+ toNumber():number {
  // -1 instead of NULL ok?
- return (this.type == TOKEN_TYPE_NUMBER) ? this._intValue : -1;
+ return (this.type == TOKEN_TYPE_NUMBER) ? this._numValue : -1;
  }
 
  toString():string {
- var type:number/*int*/ = this.type;
+ var type:int = this.type;
  if (type >= TOKEN_TYPE_CHARACTER && type <= TOKEN_TYPE_STRING) {
  return this._strValue;
  } else if (type == TOKEN_TYPE_NUMBER) {
- return this._intValue.toString();
+ return this._numValue.toString();
  } else {
  return null;
  }
  }
 }
 
-function newCharacterToken(index:number/*int*/, code:number/*int*/):Token {
+function newCharacterToken(index:int, code:int):Token {
  return new Token(index, TOKEN_TYPE_CHARACTER, code, StringWrapper.fromCharCode(code));
 }
 
-function newIdentifierToken(index:number/*int*/, text:string):Token {
+function newIdentifierToken(index:int, text:string):Token {
  return new Token(index, TOKEN_TYPE_IDENTIFIER, 0, text);
 }
 
-function newKeywordToken(index:number/*int*/, text:string):Token {
+function newKeywordToken(index:int, text:string):Token {
  return new Token(index, TOKEN_TYPE_KEYWORD, 0, text);
 }
 
-function newOperatorToken(index:number/*int*/, text:string):Token {
+function newOperatorToken(index:int, text:string):Token {
  return new Token(index, TOKEN_TYPE_OPERATOR, 0, text);
 }
 
-function newStringToken(index:number/*int*/, text:string):Token {
+function newStringToken(index:int, text:string):Token {
  return new Token(index, TOKEN_TYPE_STRING, 0, text);
 }
 
-function newNumberToken(index:number/*int*/, n:number/*int*/):Token {
+function newNumberToken(index:int, n:number):Token {
  return new Token(index, TOKEN_TYPE_NUMBER, n, "");
 }
 
@@ -211,7 +211,7 @@ export class Scanner {
  if (isIdentifierStart(peek)) return this.scanIdentifier();
  if (isDigit(peek)) return this.scanNumber(index);
 
- var start:number/*int*/ = index;
+ var start:int = index;
  switch (peek) {
  case $PERIOD:
  this.advance();
@@ -255,21 +255,21 @@ export class Scanner {
  return null;
  }
 
- scanCharacter(start:number/*int*/, code:number/*int*/):Token {
+ scanCharacter(start:int, code:int):Token {
  assert(this.peek == code);
  this.advance();
  return newCharacterToken(start, code);
  }
 
 
- scanOperator(start:number/*int*/, str:string):Token {
+ scanOperator(start:int, str:string):Token {
  assert(this.peek == StringWrapper.charCodeAt(str, 0));
  assert(SetWrapper.has(OPERATORS, str));
  this.advance();
  return newOperatorToken(start, str);
  }
 
- scanComplexOperator(start:number/*int*/, code:number/*int*/, one:string, two:string):Token {
+ scanComplexOperator(start:int, code:int, one:string, two:string):Token {
  assert(this.peek == StringWrapper.charCodeAt(one, 0));
  this.advance();
  var str:string = one;
@@ -283,7 +283,7 @@ export class Scanner {
 
  scanIdentifier():Token {
  assert(isIdentifierStart(this.peek));
- var start:number/*int*/ = this.index;
+ var start:int = this.index;
  this.advance();
  while (isIdentifierPart(this.peek)) this.advance();
  var str:string = this.input.substring(start, this.index);
@@ -294,7 +294,7 @@ export class Scanner {
  }
  }
 
- scanNumber(start:number/*int*/):Token {
+ scanNumber(start:int):Token {
  assert(isDigit(this.peek));
  var simple:boolean = (this.index === start);
  this.advance(); // Skip initial digit.
@@ -315,31 +315,31 @@ export class Scanner {
  }
  var str:string = this.input.substring(start, this.index);
  // TODO
- var value:number = simple ? NumberWrapper.parseIntAutoRadix(str) : NumberWrapper.parseDouble(str);
+ var value:number = simple ? NumberWrapper.parseIntAutoRadix(str) : NumberWrapper.parseFloat(str);
  return newNumberToken(start, value);
  }
 
  scanString():Token {
  assert(this.peek == $SQ || this.peek == $DQ);
- var start:number/*int*/ = this.index;
- var quote:number/*int*/ = this.peek;
+ var start:int = this.index;
+ var quote:int = this.peek;
  this.advance(); // Skip initial quote.
 
  var buffer:StringJoiner; //ckck
- var marker:number/*int*/ = this.index;
+ var marker:int = this.index;
  var input:string = this.input;
 
  while (this.peek != quote) {
  if (this.peek == $BACKSLASH) {
  if (buffer == null) buffer = new StringJoiner();
  buffer.add(input.substring(marker, this.index));
  this.advance();
- var unescapedCode:number/*int*/;
+ var unescapedCode:int;
  if (this.peek == $u) {
  // 4 character hex code for unicode character.
  var hex:string = input.substring(this.index + 1, this.index + 5);
  unescapedCode = NumberWrapper.parseInt(hex, 16);
- for (var i:number/*int*/ = 0; i < 5; i++) {
+ for (var i:int = 0; i < 5; i++) {
  this.advance();
  }
  } else {
@@ -369,43 +369,43 @@ export class Scanner {
  }
 
  error(message:string) {
- var position:number/*int*/ = this.index + this.offset;
+ var position:int = this.index + this.offset;
  throw `Lexer Error: ${message} at column ${position} in expression [${input}]`;
  }
 }
 
-function isWhitespace(code:number/*int*/):boolean {
+function isWhitespace(code:int):boolean {
  return (code >= $TAB && code <= $SPACE) || (code == $NBSP);
 }
 
-function isIdentifierStart(code:number/*int*/):boolean {
+function isIdentifierStart(code:int):boolean {
  return ($a <= code && code <= $z) ||
  ($A <= code && code <= $Z) ||
  (code == $_) ||
  (code == $$);
 }
 
-function isIdentifierPart(code:number/*int*/):boolean {
+function isIdentifierPart(code:int):boolean {
  return ($a <= code && code <= $z) ||
  ($A <= code && code <= $Z) ||
  ($0 <= code && code <= $9) ||
  (code == $_) ||
  (code == $$);
 }
 
-function isDigit(code:number/*int*/):boolean {
+function isDigit(code:int):boolean {
  return $0 <= code && code <= $9;
 }
 
-function isExponentStart(code:number/*int*/):boolean {
+function isExponentStart(code:int):boolean {
  return code == $e || code == $E;
 }
 
-function isExponentSign(code:number/*int*/):boolean {
+function isExponentSign(code:int):boolean {
  return code == $MINUS || code == $PLUS;
 }
 
-function unescape(code:number/*int*/):number/*int*/ {
+function unescape(code:int):int {
  switch(code) {
  case $n: return $LF;
  case $f: return $FF;
@@ -445,16 +445,4 @@ var KEYWORDS = SetWrapper.createFromList([
  'undefined',
  'true',
  'false',
-]);
-
-
-export function Lexer(text:string):List {
- var scanner:Scanner = new Scanner(text);
- var tokens:List<Token> = [];
- var token:Token = scanner.scanToken();
- while (token != null) {
- ListWrapper.push(tokens, token);
- token = scanner.scanToken();
- }
- return tokens;
-}
+]);

modules/change_detection/test/parser/lexer_spec.js

@@ -1,9 +1,20 @@
 import {describe, it, expect} from 'test_lib/test_lib';
-import {Lexer, Scanner, Token} from 'change_detection/parser/lexer';
+import {Scanner, Token} from 'change_detection/parser/scanner';
 import {DOM} from 'facade/dom';
 import {List, ListWrapper} from "facade/collection";
 import {StringWrapper} from "facade/lang";
 
+function lex(text:string):List {
+ var scanner:Scanner = new Scanner(text);
+ var tokens:List<Token> = [];
+ var token:Token = scanner.scanToken();
+ while (token != null) {
+ ListWrapper.push(tokens, token);
+ token = scanner.scanToken();
+ }
+ return tokens;
+}
+
 function expectToken(token, index) {
  expect(token instanceof Token).toBe(true);
  expect(token.index).toEqual(index);
@@ -46,51 +57,51 @@ function expectKeywordToken(token, index, keyword) {
 
 
 export function main() {
- describe('lexer', function() {
+ describe('scanner', function() {
  describe('token', function() {
  it('should tokenize a simple identifier', function() {
- var tokens:List<int> = Lexer("j");
+ var tokens:List<int> = lex("j");
  expect(tokens.length).toEqual(1);
  expectIdentifierToken(tokens[0], 0, 'j');
  });
 
  it('should tokenize a dotted identifier', function() {
- var tokens:List<int> = Lexer("j.k");
+ var tokens:List<int> = lex("j.k");
  expect(tokens.length).toEqual(3);
  expectIdentifierToken(tokens[0], 0, 'j');
  expectCharacterToken (tokens[1], 1, '.');
  expectIdentifierToken(tokens[2], 2, 'k');
  });
 
  it('should tokenize an operator', function() {
- var tokens:List<int> = Lexer("j-k");
+ var tokens:List<int> = lex("j-k");
  expect(tokens.length).toEqual(3);
  expectOperatorToken(tokens[1], 1, '-');
  });
 
  it('should tokenize an indexed operator', function() {
- var tokens:List<int> = Lexer("j[k]");
+ var tokens:List<int> = lex("j[k]");
  expect(tokens.length).toEqual(4);
  expectCharacterToken(tokens[1], 1, "[");
  expectCharacterToken(tokens[3], 3, "]");
  });
 
  it('should tokenize numbers', function() {
- var tokens:List<int> = Lexer("88");
+ var tokens:List<int> = lex("88");
  expect(tokens.length).toEqual(1);
  expectNumberToken(tokens[0], 0, 88);
  });
 
  it('should tokenize numbers within index ops', function() {
- expectNumberToken(Lexer("a[22]")[2], 2, 22);
+ expectNumberToken(lex("a[22]")[2], 2, 22);
  });
 
  it('should tokenize simple quoted strings', function() {
- expectStringToken(Lexer('"a"')[0], 0, "a");
+ expectStringToken(lex('"a"')[0], 0, "a");
  });
 
  it('should tokenize quoted strings with escaped quotes', function() {
- expectStringToken(Lexer('"a\\""')[0], 0, 'a"');
+ expectStringToken(lex('"a\\""')[0], 0, 'a"');
  });
 
  });