Parser honors :static_atoms_encoder for multi-letter sigils (#12706)

* Unify sigil token generation * Parser honors :static_atoms_encoder for multi-letter sigils * Remove chars from token, reverse engineer from atom

Author: sabiwara

lib/elixir/lib/code.ex

@@ -1136,7 +1136,10 @@ defmodule Code do
  * syntax keywords (`fn`, `do`, `else`, and so on)
 
  * atoms containing interpolation (`:"#{1 + 1} is two"`), as these
- atoms are constructed at runtime.
+ atoms are constructed at runtime
+
+ * atoms used to represent single-letter sigils like `:sigil_X`
+ (but multi-letter sigils like `:sigil_XYZ` are encoded).
 
  """
  @spec string_to_quoted(List.Chars.t(), keyword) ::

lib/elixir/src/elixir_errors.erl

@@ -170,13 +170,22 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input)
 
 %% Produce a human-readable message for errors before a sigil
 parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, Input) ->
- {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full),
+ {sigil, _, Atom, [Content | _], _, _, _} = parse_erl_term(Full),
  Content2 = case is_binary(Content) of
  true -> Content;
  false -> <<>>
  end,
- SigilName = list_to_binary(Sigil),
- Message = <<"syntax error before: sigil \~", SigilName/binary, " starting with content '", Content2/binary, "'">>,
+
+ % :static_atoms_encoder might encode :sigil_ atoms as arbitrary terms
+ MaybeSigil = case is_atom(Atom) of
+ true -> case atom_to_binary(Atom) of
+ <<"sigil_", Chars/binary>> -> <<"\~", Chars/binary, " ">>;
+ _ -> <<>>
+ end;
+ false -> <<>>
+ end,
+
+ Message = <<"syntax error before: sigil ", MaybeSigil/binary, "starting with content '", Content2/binary, "'">>,
  raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message);
 
 %% Binaries (and interpolation) are wrapped in [<<...>>]

lib/elixir/src/elixir_parser.yrl

@@ -939,11 +939,11 @@ build_access(Expr, {List, Meta}) ->
 
 %% Interpolation aware
 
-build_sigil({sigil, Location, Sigil, Parts, Modifiers, Indentation, Delimiter}) ->
+build_sigil({sigil, Location, Atom, Parts, Modifiers, Indentation, Delimiter}) ->
  Meta = meta_from_location(Location),
  MetaWithDelimiter = [{delimiter, Delimiter} | Meta],
  MetaWithIndentation = meta_with_indentation(Meta, Indentation),
- {list_to_atom("sigil_" ++ Sigil),
+ {Atom,
  MetaWithDelimiter,
  [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}.
 

lib/elixir/src/elixir_tokenizer.erl

@@ -1550,7 +1550,7 @@ tokenize_sigil_name([S | T], NameAcc, Line, Column, Scope, Tokens) when ?is_upca
  tokenize_sigil_name(T, [S | NameAcc], Line, Column + 1, Scope, Tokens);
 % With a lowercase letter and a non-empty NameAcc we return an error.
 tokenize_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) ->
- Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ 
+ Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++
  " sequence of uppercase letters only, got: ",
  {error, Message, [$~] ++ lists:reverse(NameAcc) ++ Original};
 % We finished the letters, so the name is over.
@@ -1561,12 +1561,8 @@ tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Col
  when ?is_quote(H) ->
  case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of
  {ok, NewLine, NewColumn, Parts, Rest, NewScope} ->
- {Final, Modifiers} = collect_modifiers(Rest, []),
  Indentation = NewColumn - 4,
- TokenColumn = Column - 1 - length(SigilName),
- Token = {sigil, {Line, TokenColumn, nil}, SigilName, Parts, Modifiers, Indentation, <<H, H, H>>},
- NewColumnWithModifiers = NewColumn + length(Modifiers),
- tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]);
+ add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <<H, H, H>>);
 
  {error, Reason} ->
  error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens)
@@ -1576,12 +1572,8 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S
  when ?is_sigil(H) ->
  case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of
  {NewLine, NewColumn, Parts, Rest, NewScope} ->
- {Final, Modifiers} = collect_modifiers(Rest, []),
  Indentation = nil,
- TokenColumn = Column - 1 - length(SigilName),
- Token = {sigil, {Line, TokenColumn, nil}, SigilName, tokens_to_binary(Parts), Modifiers, Indentation, <<H>>},
- NewColumnWithModifiers = NewColumn + length(Modifiers),
- tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]);
+ add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <<H>>);
 
  {error, Reason} ->
  Sigil = [$~, S, H],
@@ -1601,6 +1593,24 @@ tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Toke
 tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) ->
  tokenize([], Line, Column, Scope, Tokens).
 
+add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) ->
+ TokenColumn = Column - 1 - length(SigilName),
+ MaybeEncoded = case SigilName of
+ % Single-letter sigils present no risk of atom exhaustion (limited possibilities)
+ [_Char] -> {ok, list_to_atom("sigil_" ++ SigilName)};
+ _ -> unsafe_to_atom("sigil_" ++ SigilName, Line, TokenColumn, Scope)
+ end,
+ case MaybeEncoded of
+ {ok, Atom} ->
+ {Final, Modifiers} = collect_modifiers(Rest, []),
+ Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter},
+ NewColumnWithModifiers = NewColumn + length(Modifiers),
+ tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]);
+
+ {error, Reason} ->
+ error(Reason, Rest, Scope, Tokens)
+ end.
+
 %% Fail early on invalid do syntax. For example, after
 %% most keywords, after comma and so on.
 tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) ->

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -182,6 +182,9 @@ defmodule Kernel.ParserTest do
  assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) ==
  {:error,
  {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}}
+
+ assert Code.string_to_quoted("~UNKNOWN'foo bar'", existing_atoms_only: true) ==
+ {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "sigil_UNKNOWN"}}
  end
 
  test "encodes atoms" do
@@ -228,6 +231,20 @@ defmodule Kernel.ParserTest do
  )
  end
 
+ test "encodes multi-letter sigils" do
+ ref = make_ref()
+
+ encoder = fn atom, meta ->
+ assert atom == "sigil_UNKNOWN"
+ assert meta[:line] == 1
+ assert meta[:column] == 1
+ {:ok, ref}
+ end
+
+ assert {:ok, {^ref, [delimiter: "'", line: 1], [{:<<>>, [line: 1], ["abc"]}, []]}} =
+ Code.string_to_quoted("~UNKNOWN'abc'", static_atoms_encoder: encoder)
+ end
+
  test "addresses ambiguities" do
  encoder = fn string, _meta -> {:ok, {:atom, string}} end
 
@@ -254,6 +271,16 @@ defmodule Kernel.ParserTest do
  Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder)
  end
 
+ test "does not encode one-letter sigils" do
+ encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end
+
+ assert {:ok, {:sigil_z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} =
+ Code.string_to_quoted("~z'foo'", static_atoms_encoder: encoder)
+
+ assert {:ok, {:sigil_Z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} =
+ Code.string_to_quoted("~Z'foo'", static_atoms_encoder: encoder)
+ end
+
  test "returns errors on long atoms even when using static_atoms_encoder" do
  atom = String.duplicate("a", 256)
 
@@ -271,6 +298,9 @@ defmodule Kernel.ParserTest do
 
  assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} =
  Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder)
+
+ assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "sigil_UNKNOWN"}} =
+ Code.string_to_quoted("~UNKNOWN'foo bar'", static_atoms_encoder: encoder)
  end
 
  test "may return tuples" do

lib/elixir/test/erlang/tokenizer_test.erl

@@ -261,22 +261,22 @@ vc_merge_conflict_test() ->
  tokenize_error("<<<<<<< HEAD\n[1, 2, 3]").
 
 sigil_terminator_test() ->
- [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"),
- [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"),
- [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""),
- [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>},
+ [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"),
+ [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"),
+ [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""),
+ [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>},
  {comp_op, {1, 9, nil}, '=='},
  {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"),
- [{sigil, {1, 1, nil}, "r", [<<"foo">>], "iu", nil, <<"/">>},
+ [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>},
  {comp_op, {1, 11, nil}, '=='},
  {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"),
- [{sigil, {1, 1, nil}, "M", [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
+ [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
 
 sigil_heredoc_test() ->
- [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
- [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
- [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""),
- [{sigil, {1, 1, nil}, "s", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\"").
+ [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
+ [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
+ [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""),
+ [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\"").
 
 invalid_sigil_delimiter_test() ->
  {1, 1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"),