Parser honors :static_atoms_encoder for multi-letter sigils (#12706)

* Unify sigil token generation

* Parser honors :static_atoms_encoder for multi-letter sigils

* Remove chars from token, reverse engineer from atom

Author: sabiwara

lib/elixir/lib/code.ex

@@ -1136,7 +1136,10 @@ defmodule Code do
     * syntax keywords (`fn`, `do`, `else`, and so on)
 
     * atoms containing interpolation (`:"#{1 + 1} is two"`), as these
-      atoms are constructed at runtime.
+      atoms are constructed at runtime
+
+    * atoms used to represent single-letter sigils like `:sigil_X`
+      (but multi-letter sigils like `:sigil_XYZ` are encoded).
 
   """
   @spec string_to_quoted(List.Chars.t(), keyword) ::

lib/elixir/src/elixir_errors.erl

@@ -170,13 +170,22 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input)
 
 %% Produce a human-readable message for errors before a sigil
 parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, Input) ->
-  {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full),
+  {sigil, _, Atom, [Content | _], _, _, _} = parse_erl_term(Full),
   Content2 = case is_binary(Content) of
     true -> Content;
     false -> <<>>
   end,
-  SigilName = list_to_binary(Sigil),
-  Message = <<"syntax error before: sigil \~", SigilName/binary, " starting with content '", Content2/binary, "'">>,
+
+  % :static_atoms_encoder might encode :sigil_ atoms as arbitrary terms
+  MaybeSigil = case is_atom(Atom) of
+    true -> case atom_to_binary(Atom) of
+      <<"sigil_", Chars/binary>> -> <<"\~", Chars/binary, " ">>;
+      _ -> <<>>
+    end;
+    false -> <<>>
+  end,
+
+  Message = <<"syntax error before: sigil ", MaybeSigil/binary, "starting with content '", Content2/binary, "'">>,
   raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message);
 
 %% Binaries (and interpolation) are wrapped in [<<...>>]

lib/elixir/src/elixir_parser.yrl

@@ -939,11 +939,11 @@ build_access(Expr, {List, Meta}) ->
 
 %% Interpolation aware
 
-build_sigil({sigil, Location, Sigil, Parts, Modifiers, Indentation, Delimiter}) ->
+build_sigil({sigil, Location, Atom, Parts, Modifiers, Indentation, Delimiter}) ->
   Meta = meta_from_location(Location),
   MetaWithDelimiter = [{delimiter, Delimiter} | Meta],
   MetaWithIndentation = meta_with_indentation(Meta, Indentation),
-  {list_to_atom("sigil_" ++ Sigil),
+  {Atom,
    MetaWithDelimiter,
    [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}.
 

lib/elixir/src/elixir_tokenizer.erl

@@ -1550,7 +1550,7 @@ tokenize_sigil_name([S | T], NameAcc, Line, Column, Scope, Tokens) when ?is_upca
   tokenize_sigil_name(T, [S | NameAcc], Line, Column + 1, Scope, Tokens);
 % With a lowercase letter and a non-empty NameAcc we return an error.
 tokenize_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) ->
-  Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ 
+  Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++
             " sequence of uppercase letters only, got: ",
   {error, Message, [$~] ++ lists:reverse(NameAcc) ++ Original};
 % We finished the letters, so the name is over.
@@ -1561,12 +1561,8 @@ tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Col
     when ?is_quote(H) ->
   case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of
     {ok, NewLine, NewColumn, Parts, Rest, NewScope} ->
-      {Final, Modifiers} = collect_modifiers(Rest, []),
       Indentation = NewColumn - 4,
-      TokenColumn = Column - 1 - length(SigilName),
-      Token = {sigil, {Line, TokenColumn, nil}, SigilName, Parts, Modifiers, Indentation, <<H, H, H>>},
-      NewColumnWithModifiers = NewColumn + length(Modifiers),
-      tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]);
+      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <<H, H, H>>);
 
     {error, Reason} ->
       error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens)
@@ -1576,12 +1572,8 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S
     when ?is_sigil(H) ->
   case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of
     {NewLine, NewColumn, Parts, Rest, NewScope} ->
-      {Final, Modifiers} = collect_modifiers(Rest, []),
       Indentation = nil,
-      TokenColumn = Column - 1 - length(SigilName),
-      Token = {sigil, {Line, TokenColumn, nil}, SigilName, tokens_to_binary(Parts), Modifiers, Indentation, <<H>>},
-      NewColumnWithModifiers = NewColumn + length(Modifiers),
-      tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]);
+      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <<H>>);
 
     {error, Reason} ->
       Sigil = [$~, S, H],
@@ -1601,6 +1593,24 @@ tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Toke
 tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) ->
   tokenize([], Line, Column, Scope, Tokens).
 
+add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) ->
+  TokenColumn = Column - 1 - length(SigilName),
+  MaybeEncoded = case SigilName of
+    % Single-letter sigils present no risk of atom exhaustion (limited possibilities)
+    [_Char] -> {ok, list_to_atom("sigil_" ++ SigilName)};
+    _ -> unsafe_to_atom("sigil_" ++ SigilName, Line, TokenColumn, Scope)
+  end,
+  case MaybeEncoded of
+    {ok, Atom} ->
+      {Final, Modifiers} = collect_modifiers(Rest, []),
+      Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter},
+      NewColumnWithModifiers = NewColumn + length(Modifiers),
+      tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]);
+
+    {error, Reason} ->
+      error(Reason, Rest, Scope, Tokens)
+  end.
+
 %% Fail early on invalid do syntax. For example, after
 %% most keywords, after comma and so on.
 tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) ->

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -182,6 +182,9 @@ defmodule Kernel.ParserTest do
       assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) ==
                {:error,
                 {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}}
+
+      assert Code.string_to_quoted("~UNKNOWN'foo bar'", existing_atoms_only: true) ==
+               {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "sigil_UNKNOWN"}}
     end
 
     test "encodes atoms" do
@@ -228,6 +231,20 @@ defmodule Kernel.ParserTest do
                )
     end
 
+    test "encodes multi-letter sigils" do
+      ref = make_ref()
+
+      encoder = fn atom, meta ->
+        assert atom == "sigil_UNKNOWN"
+        assert meta[:line] == 1
+        assert meta[:column] == 1
+        {:ok, ref}
+      end
+
+      assert {:ok, {^ref, [delimiter: "'", line: 1], [{:<<>>, [line: 1], ["abc"]}, []]}} =
+               Code.string_to_quoted("~UNKNOWN'abc'", static_atoms_encoder: encoder)
+    end
+
     test "addresses ambiguities" do
       encoder = fn string, _meta -> {:ok, {:atom, string}} end
 
@@ -254,6 +271,16 @@ defmodule Kernel.ParserTest do
                Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder)
     end
 
+    test "does not encode one-letter sigils" do
+      encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end
+
+      assert {:ok, {:sigil_z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} =
+               Code.string_to_quoted("~z'foo'", static_atoms_encoder: encoder)
+
+      assert {:ok, {:sigil_Z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} =
+               Code.string_to_quoted("~Z'foo'", static_atoms_encoder: encoder)
+    end
+
     test "returns errors on long atoms even when using static_atoms_encoder" do
       atom = String.duplicate("a", 256)
 
@@ -271,6 +298,9 @@ defmodule Kernel.ParserTest do
 
       assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} =
                Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder)
+
+      assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "sigil_UNKNOWN"}} =
+               Code.string_to_quoted("~UNKNOWN'foo bar'", static_atoms_encoder: encoder)
     end
 
     test "may return tuples" do

lib/elixir/test/erlang/tokenizer_test.erl

@@ -261,22 +261,22 @@ vc_merge_conflict_test() ->
     tokenize_error("<<<<<<< HEAD\n[1, 2, 3]").
 
 sigil_terminator_test() ->
-  [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"),
-  [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"),
-  [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""),
-  [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>},
+  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"),
+  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"),
+  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""),
+  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>},
    {comp_op, {1, 9, nil}, '=='},
    {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"),
-  [{sigil, {1, 1, nil}, "r", [<<"foo">>], "iu", nil, <<"/">>},
+  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>},
    {comp_op, {1, 11, nil}, '=='},
    {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"),
-  [{sigil, {1, 1, nil}, "M", [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
+  [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
 
 sigil_heredoc_test() ->
-  [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
-  [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
-  [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n  sigil heredoc\n  \"\"\""),
-  [{sigil, {1, 1, nil}, "s", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n  sigil heredoc\n  \"\"\"").
+  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
+  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
+  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n  sigil heredoc\n  \"\"\""),
+  [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n  sigil heredoc\n  \"\"\"").
 
 invalid_sigil_delimiter_test() ->
   {1, 1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"),