Normalize token missing and mismatched delimiter exceptions

Closes #13183.
Closes #13185.
Closes #13186.
Closes #13187.

Author: josevalim

lib/elixir/lib/exception.ex

@@ -795,12 +795,10 @@ defmodule Exception do
   end
 
   @doc false
-  def format_expected_delimiter(opening_delimiter) do
-    terminator = :elixir_tokenizer.terminator(opening_delimiter)
-
-    if terminator |> Atom.to_string() |> String.contains?(["\"", "'"]),
-      do: terminator,
-      else: ~s("#{terminator}")
+  def format_delimiter(delimiter) do
+    if delimiter |> Atom.to_string() |> String.contains?(["\"", "'"]),
+      do: delimiter,
+      else: ~s("#{delimiter}")
   end
 
   @doc false
@@ -1159,8 +1157,23 @@ defmodule MismatchedDelimiterError do
   An exception raised when a mismatched delimiter is found when parsing code.
 
   For example:
-  - `[1, 2, 3}`
-  - `fn a -> )`
+
+    * `[1, 2, 3}`
+    * `fn a -> )`
+
+  The following fields of this exceptions are public and can be accessed freely:
+
+    * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if
+      the error occurred in code that did not come from a file
+    * `:line` - the line for the opening delimiter
+    * `:column` - the column for the opening delimiter
+    * `:end_line` - the line for the mismatched closing delimiter
+    * `:end_column` - the column for the mismatched closing delimiter
+    * `:opening_delimiter` - an atom representing the opening delimiter
+    * `:closing_delimiter` - an atom representing the mismatched closing delimiter
+    * `:expected_delimiter` - an atom representing the closing delimiter
+    * `:description` - a description of the mismatched delimiter error
+
   """
 
   defexception [
@@ -1172,6 +1185,7 @@ defmodule MismatchedDelimiterError do
     :end_column,
     :opening_delimiter,
     :closing_delimiter,
+    :expected_delimiter,
     :snippet,
     description: "mismatched delimiter error"
   ]
@@ -1184,15 +1198,14 @@ defmodule MismatchedDelimiterError do
         end_column: end_column,
         line_offset: line_offset,
         description: description,
-        opening_delimiter: opening_delimiter,
-        closing_delimiter: _closing_delimiter,
+        expected_delimiter: expected_delimiter,
         file: file,
         snippet: snippet
       }) do
     start_pos = {start_line, start_column}
     end_pos = {end_line, end_column}
     lines = String.split(snippet, "\n")
-    expected_delimiter = Exception.format_expected_delimiter(opening_delimiter)
+    expected_delimiter = Exception.format_delimiter(expected_delimiter)
 
     start_message = "└ unclosed delimiter"
     end_message = ~s/└ mismatched closing delimiter (expected #{expected_delimiter})/
@@ -1226,8 +1239,9 @@ defmodule SyntaxError do
 
     * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if
       the error occurred in code that did not come from a file
-    * `:line` (`t:non_neg_integer/0`) - the line where the error occurred
-    * `:column` (`t:non_neg_integer/0`) - the column where the error occurred
+    * `:line` - the line where the error occurred
+    * `:column` - the column where the error occurred
+    * `:description` - a description of the syntax error
 
   """
 
@@ -1276,19 +1290,25 @@ defmodule TokenMissingError do
 
     * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if
       the error occurred in code that did not come from a file
-    * `:line` (`t:non_neg_integer/0`) - the line where the error occurred
-    * `:column` (`t:non_neg_integer/0`) - the column where the error occurred
-
+    * `:line` - the line for the opening delimiter
+    * `:column` - the column for the opening delimiter
+    * `:end_line` - the line for the end of the string
+    * `:end_column` - the column for the end of the string
+    * `:opening_delimiter` - an atom representing the opening delimiter
+    * `:expected_delimiter` - an atom representing the expected delimiter
+    * `:description` - a description of the missing token error
   """
 
   defexception [
     :file,
     :line,
     :column,
     :end_line,
+    :end_column,
     :line_offset,
     :snippet,
     :opening_delimiter,
+    :expected_delimiter,
     description: "expression is incomplete"
   ]
 
@@ -1300,7 +1320,7 @@ defmodule TokenMissingError do
         end_line: end_line,
         line_offset: line_offset,
         description: description,
-        opening_delimiter: opening_delimiter,
+        expected_delimiter: expected_delimiter,
         snippet: snippet
       })
       when not is_nil(snippet) and not is_nil(column) and not is_nil(end_line) do
@@ -1315,7 +1335,7 @@ defmodule TokenMissingError do
 
     start_pos = {line, column}
     end_pos = {end_line, end_column}
-    expected_delimiter = Exception.format_expected_delimiter(opening_delimiter)
+    expected_delimiter = Exception.format_delimiter(expected_delimiter)
 
     start_message = ~s/└ unclosed delimiter/
     end_message = ~s/└ missing closing delimiter (expected #{expected_delimiter})/

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -890,7 +890,7 @@ defmodule Kernel.ParallelCompiler do
   end
 
   defp to_error(file, kind, reason, stack) do
-    line = get_line(file, reason, stack)
+    {line, span} = get_line_span(file, reason, stack)
     file = Path.absname(file)
     message = :unicode.characters_to_binary(Kernel.CLI.format_error(kind, reason, stack))
 
@@ -900,43 +900,55 @@ defmodule Kernel.ParallelCompiler do
       message: message,
       severity: :error,
       stacktrace: stack,
-      span: nil,
+      span: span,
       exception: get_exception(reason)
     }
   end
 
   defp get_exception(exception) when is_exception(exception), do: exception
   defp get_exception(_reason), do: nil
 
-  defp get_line(_file, %{line: line, column: column}, _stack)
+  defp get_line_span(
+         _file,
+         %{line: line, column: column, end_line: end_line, end_column: end_column},
+         _stack
+       )
+       when is_integer(line) and line > 0 and is_integer(column) and column >= 0 and
+              is_integer(end_line) and end_line > 0 and is_integer(end_column) and end_column >= 0 do
+    {{line, column}, {end_line, end_column}}
+  end
+
+  defp get_line_span(_file, %{line: line, column: column}, _stack)
        when is_integer(line) and line > 0 and is_integer(column) and column >= 0 do
-    {line, column}
+    {{line, column}, nil}
   end
 
-  defp get_line(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do
-    line
+  defp get_line_span(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do
+    {line, nil}
   end
 
-  defp get_line(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do
-    if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do
-      Keyword.get(info, :line)
-    end
+  defp get_line_span(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do
+    get_line_span_from_stacktrace_info(info, file)
   end
 
-  defp get_line(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _])
+  defp get_line_span(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _])
        when expanding in [~c"expanding macro", ~c"expanding struct"] do
-    if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do
-      Keyword.get(info, :line)
-    end
+    get_line_span_from_stacktrace_info(info, file)
   end
 
-  defp get_line(file, _reason, [{_, _, _, info} | _]) do
-    if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do
-      Keyword.get(info, :line)
-    end
+  defp get_line_span(file, _reason, [{_, _, _, info} | _]) do
+    get_line_span_from_stacktrace_info(info, file)
+  end
+
+  defp get_line_span(_, _, _) do
+    {nil, nil}
   end
 
-  defp get_line(_, _, _) do
-    nil
+  defp get_line_span_from_stacktrace_info(info, file) do
+    if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do
+      {Keyword.get(info, :line), nil}
+    else
+      {nil, nil}
+    end
   end
 end

lib/elixir/src/elixir_tokenizer.erl

@@ -142,18 +142,20 @@ tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope,
   AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens),
   {ok, Line, Column, AllWarnings, AccTokens};
 
-tokenize([], EndLine, _, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) ->
+tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) ->
   End = terminator(Start),
   Hint = missing_terminator_hint(Start, End, Scope),
   Message = "missing terminator: ~ts",
   Formatted = io_lib:format(Message, [End]),
   Meta = [
-          {error_type, unclosed_delimiter},
-          {opening_delimiter, Start},
-          {line, StartLine},
-          {column, StartColumn},
-          {end_line, EndLine}
-         ],
+    {error_type, unclosed_delimiter},
+    {opening_delimiter, Start},
+    {expected_delimiter, End},
+    {line, StartLine},
+    {column, StartColumn},
+    {end_line, EndLine},
+    {end_column, EndColumn}
+  ],
   error({Meta, [Formatted, Hint], []}, [], Scope, Tokens);
 
 tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) ->
@@ -531,7 +533,7 @@ tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H)
 
     {error, Reason} ->
       Message = " (for atom starting at line ~B)",
-      interpolation_error(Reason, Original, Scope, Tokens, Message, [Line])
+      interpolation_error(Reason, Original, Scope, Tokens, Message, [Line], Line, Column + 1, [H], [H])
   end;
 
 tokenize([$: | String] = Original, Line, Column, Scope, Tokens) ->
@@ -772,7 +774,7 @@ handle_heredocs(T, Line, Column, H, Scope, Tokens) ->
 handle_strings(T, Line, Column, H, Scope, Tokens) ->
   case elixir_interpolation:extract(Line, Column, Scope, true, T, H) of
     {error, Reason} ->
-      interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line]);
+      interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line], Line, Column-1, [H], [H]);
 
     {NewLine, NewColumn, Parts, [$: | Rest], InterScope} when ?is_space(hd(Rest)) ->
       NewScope = case is_unnecessary_quote(Parts, InterScope) of
@@ -925,7 +927,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?i
       Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ",
       error({?LOC(Line, Column), Message, [H]}, Rest, NewScope, Tokens);
     {error, Reason} ->
-      interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line])
+      interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line], Line, Column, [H], [H])
   end;
 
 handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) ->
@@ -1030,16 +1032,7 @@ extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) ->
           {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope};
 
         {error, Reason} ->
-          {Position, Message, List} = interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line]),
-          {line, EndLine} = lists:keyfind(line, 1, Position),
-           Meta = [
-             {error_type, unclosed_delimiter},
-             {opening_delimiter, '"""'},
-             {line, Line},
-             {column, Column},
-             {end_line, EndLine}
-          ],
-          {error, {Meta, Message, List}}
+          {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line], Line, Column, [H, H, H], [H, H, H])}
       end;
 
     error ->
@@ -1352,12 +1345,21 @@ previous_was_eol(_) -> nil.
 
 %% Error handling
 
-interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args) ->
-  error(interpolation_format(Reason, Extension, Args), Rest, Scope, Tokens).
+interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args, Line, Column, Opening, Closing) ->
+  error(interpolation_format(Reason, Extension, Args, Line, Column, Opening, Closing), Rest, Scope, Tokens).
 
-interpolation_format({string, Line, Column, Message, Token}, Extension, Args) ->
-  {?LOC(Line, Column), [Message, io_lib:format(Extension, Args)], Token};
-interpolation_format({_, _, _} = Reason, _Extension, _Args) ->
+interpolation_format({string, EndLine, EndColumn, Message, Token}, Extension, Args, Line, Column, Opening, Closing) ->
+  Meta = [
+    {error_type, unclosed_delimiter},
+    {opening_delimiter, list_to_atom(Opening)},
+    {expected_delimiter, list_to_atom(Closing)},
+    {line, Line},
+    {column, Column},
+    {end_line, EndLine},
+    {end_column, EndColumn}
+  ],
+  {Meta, [Message, io_lib:format(Extension, Args)], Token};
+interpolation_format({_, _, _} = Reason, _Extension, _Args, _Line, _Column, _Opening, _Closing) ->
   Reason.
 
 %% Terminators
@@ -1429,15 +1431,16 @@ check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColum
     End ->
       {ok, Scope#elixir_tokenizer{terminators=Terminators}};
 
-    _ExpectedEnd ->
+    ExpectedEnd ->
       Meta = [
         {line, StartLine},
         {column, StartColumn},
         {end_line, EndLine},
         {end_column, EndColumn},
         {error_type, mismatched_delimiter},
         {opening_delimiter, Start},
-        {closing_delimiter, End}
+        {closing_delimiter, End},
+        {expected_delimiter, ExpectedEnd}
      ],
      {error, {Meta, unexpected_token_or_reserved(End), [atom_to_list(End)]}}
   end;
@@ -1490,7 +1493,6 @@ terminator('do') -> 'end';
 terminator('(')  -> ')';
 terminator('[')  -> ']';
 terminator('{')  -> '}';
-terminator('"""') -> '"""';
 terminator('<<') -> '>>'.
 
 %% Keywords checking
@@ -1596,7 +1598,7 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S
     {error, Reason} ->
       Sigil = [$~, S, H],
       Message = " (for sigil ~ts starting at line ~B)",
-      interpolation_error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens, Message, [Sigil, Line])
+      interpolation_error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens, Message, [Sigil, Line], Line, Column, [H], [sigil_terminator(H)])
   end;
 
 tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Tokens) ->

lib/elixir/test/elixir/code_test.exs

@@ -498,9 +498,13 @@ defmodule CodeTest do
   end
 
   test "string_to_quoted returns error on incomplete escaped string" do
-    assert Code.string_to_quoted("\"\\") ==
-             {:error,
-              {[line: 1, column: 3], "missing terminator: \" (for string starting at line 1)", ""}}
+    assert {:error, {meta, "missing terminator: \" (for string starting at line 1)", ""}} =
+             Code.string_to_quoted("\"\\")
+
+    assert meta[:line] == 1
+    assert meta[:column] == 1
+    assert meta[:end_line] == 1
+    assert meta[:end_column] == 3
   end
 
   test "compile source" do