Merge pull request #4 from nkovacs/better-function-param-parsing

Improve function param parsing

Author: eemeli

README.md

@@ -4,13 +4,23 @@ A [PEG.js] parser for [ICU MessageFormat] strings – part of [messageformat.js]
 Outputs an AST defined by [parser.pegjs].
 
 The generated parser function takes two parameters, first the string to be
-parsed, and a second optional parameter `options`, an object containing arrays
+parsed, and a second optional parameter `options`, an object.
+
+The options object contains arrays
 of keywords for `cardinal` and `ordinal` rules for the current locale – these
 are used to validate plural and selectordinal keys. If `options` or its fields
 are missing or set to false, the full set of valid [Unicode CLDR] keys is used:
 `'zero', 'one', 'two', 'few', 'many', 'other'`. To disable this check, pass in
 an empty array.
 
+The `options` object also supports a setting that makes the parser
+follow the ICU MessageFormat spec more closely: `strictFunctionParams`.
+
+By default, function parameters are split on commas and trimmed,
+so the parameters in `{x,fn, a, b }` are parsed as `['a','b']`.
+Setting `strictFunctionParams` to true will result in a params array
+with a single element: `[' a, b ']`.
+
 [ICU MessageFormat]: https://messageformat.github.io/guide/
 [messageformat.js]: https://messageformat.github.io/
 [parser.pegjs]: ./parser.pegjs

parser.pegjs

@@ -1,4 +1,4 @@
-start = token* 
+start = token*
 
 token
  = argument / select / plural / function
@@ -37,7 +37,7 @@ plural = '{' _ arg:id _ ',' _ type:('plural'/'selectordinal') _ ',' _ offset:off
  };
  }
 
-function = '{' _ arg:id _ ',' _ key:id _ params:functionParams* '}' {
+function = '{' _ arg:id _ ',' _ key:id _ params:functionParams '}' {
  return {
  type: 'function',
  arg: arg,
@@ -48,6 +48,10 @@ function = '{' _ arg:id _ ',' _ key:id _ params:functionParams* '}' {
 
 id = $([0-9a-zA-Z$_][^ \t\n\r,.+={}]*)
 
+paramDefault = str:paramcharsDefault+ { return str.join(''); }
+
+paramStrict = str:paramcharsStrict+ { return str.join(''); }
+
 selectCase = _ key:id _ tokens:caseTokens { return { key: key, tokens: tokens }; }
 
 pluralCase = _ key:pluralKey _ tokens:caseTokens { return { key: key, tokens: tokens }; }
@@ -60,7 +64,25 @@ pluralKey
  = id
  / '=' d:digits { return d; }
 
-functionParams = _ ',' _ p:id _ { return p; }
+functionParams
+ = p:functionParamsDefault* ! { return options.strictFunctionParams; } { return p; }
+ / p:functionParamsStrict* & { return options.strictFunctionParams; } { return p; }
+
+functionParamsStrict = _ ',' p:paramStrict { return p; }
+
+functionParamsDefault = _ ',' _ p:paramDefault _ { return p.replace(/^[ \t\n\r]*|[ \t\n\r]*$/g, ''); }
+
+doubleapos = "''" { return "'"; }
+
+inapos = doubleapos / str:[^']+ { return str.join(''); }
+
+quotedCurly
+ = "'{"str:inapos*"'" { return '\u007B'+str.join(''); }
+ / "'}"str:inapos*"'" { return '\u007D'+str.join(''); }
+
+quotedFunctionParams
+ = quotedCurly
+ / "'"
 
 char
  = [^{}#\\\0-\x08\x0e-\x1f\x7f]
@@ -72,6 +94,18 @@ char
  return String.fromCharCode(parseInt('0x' + h1 + h2 + h3 + h4));
  }
 
+paramcharsCommon
+ = doubleapos
+ / quotedFunctionParams
+
+paramcharsDefault
+ = paramcharsCommon
+ / str:[^',}]+ { return str.join(''); }
+
+paramcharsStrict
+ = paramcharsCommon
+ / str:[^'}]+ { return str.join(''); }
+
 digits = $([0-9]+)
 
 hexDigit = [0-9a-fA-F]

test.js

@@ -239,6 +239,61 @@ describe("Ordinals", function() {
  });
 
 });
+describe("Functions", function() {
+ it("should accept no parameters", function() {
+ expect(parse('{var,date}')[0].type).to.eql('function');
+ expect(parse('{var,date}')[0].key).to.eql('date');
+ expect(parse('{var,date}')[0].params).to.be.empty;
+ })
+
+ it("should accept parameters", function() {
+ expect(parse('{var,date,long}')[0].type).to.eql('function');
+ expect(parse('{var,date,long}')[0].key).to.eql('date');
+ expect(parse('{var,date,long}')[0].params[0]).to.eql('long');
+ expect(parse('{var,date,long,short}')[0].params[0]).to.eql('long');
+ expect(parse('{var,date,long,short}')[0].params[1]).to.eql('short');
+ })
+
+ it("should accept parameters with whitespace", function() {
+ expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].type).to.eql('function');
+ expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].key).to.eql('date');
+ expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].params[0]).to.eql('y-M-d HH:mm:ss zzzz');
+ // This is not how ICU works. ICU does not trim whitespace,
+ // but messageformat-parse must trim it to maintain backwards compatibility.
+ expect(parse('{var,date, y-M-d HH:mm:ss zzzz }')[0].params[0]).to.eql('y-M-d HH:mm:ss zzzz');
+ // This is how ICU works.
+ expect(parse('{var,date, y-M-d HH:mm:ss zzzz }', { strictFunctionParams: true })[0].params[0]).to.eql(' y-M-d HH:mm:ss zzzz ');
+ })
+
+ it("should accept parameters with special characters", function() {
+ expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].type).to.eql('function');
+ expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].key).to.eql('date');
+ expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d {,} ' HH:mm:ss zzzz");
+ expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz'}'}")[0].params[0]).to.eql("y-M-d {,} ' HH:mm:ss zzzz}");
+ expect(parse("{var,date,y-M-d # HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d # HH:mm:ss zzzz");
+ expect(parse("{var,date,y-M-d '#' HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d '#' HH:mm:ss zzzz");
+ // This is not how ICU works.
+ expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d");
+ expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}")[0].params[1]).to.eql("HH:mm:ss zzzz");
+ // This is how ICU works, but this only allows a single argStyle parameter.
+ expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}", { strictFunctionParams: true })[0].params[0]).to.eql("y-M-d, HH:mm:ss zzzz");
+ })
+
+ it("should be gracious with whitespace", function() {
+ var firstRes = JSON.stringify(parse('{var, date, long, short}'));
+ expect(JSON.stringify(parse('{ var, date, long, short }'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{var,date,long,short}'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{\nvar, \ndate,\n long\n\n,\n short\n\n\n}'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{\tvar\t,\t\t\r date\t\n, \tlong\n, short\t\n\n\n\n}'))).to.eql(firstRes);
+
+ // This is not how ICU works. ICU does not trim whitespace.
+ firstRes = JSON.stringify(parse('{var, date, y-M-d HH:mm:ss zzzz}'));
+ expect(JSON.stringify(parse('{ var, date, y-M-d HH:mm:ss zzzz }'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{var,date,y-M-d HH:mm:ss zzzz}'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{\nvar, \ndate,\n \n\n\n y-M-d HH:mm:ss zzzz\n\n\n}'))).to.eql(firstRes);
+ expect(JSON.stringify(parse('{\tvar\t,\t\t\r date\t\n, \t\ny-M-d HH:mm:ss zzzz\t\n\n\n\n}'))).to.eql(firstRes);
+ });
+});
 describe("Nested/Recursive blocks", function() {
 
  it("should allow a select statement inside of a select statement", function() {