Track source position information in Streams

Prior to this change, the only positional variables which were tracked
when moving through the stream was the current index in the stream.

We would like to start reporting richer positional information from the
parser, specifically the position (in terms of row/column) in the source
object.

This change allows this by updating the Stream class to additionally
track its current (zero-indexed) row and column while moving through the
stream. We will persist this information into the syntax tree in the
following commits.

Author: leamingrad

fluent.syntax/fluent/syntax/stream.py

@@ -9,7 +9,11 @@ class ParserStream:
     def __init__(self, string: str):
         self.string = string
         self.index = 0
+        self.row_index = 0
+        self.column_index = 0
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
 
     def get(self, offset: int) -> Union[str, None]:
         try:
@@ -37,9 +41,19 @@ def current_peek(self) -> Union[str, None]:
 
     def next(self) -> Union[str, None]:
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
+
         # Skip over CRLF as if it was a single character.
         if self.get(self.index) == "\r" and self.get(self.index + 1) == "\n":
             self.index += 1
+        # If we have reached a newline reset the position
+        if self.get(self.index) == "\n":
+            self.row_index += 1
+            self.column_index = 0
+        else:
+            self.column_index += 1
+
         self.index += 1
         return self.get(self.index)
 
@@ -50,6 +64,13 @@ def peek(self) -> Union[str, None]:
             and self.get(self.index + self.peek_offset + 1) == "\n"
         ):
             self.peek_offset += 1
+
+        if self.get(self.index + self.peek_offset) == "\n":
+            self._peek_row_offset += 1
+            self._peek_column_offset = 0
+        else:
+            self._peek_column_offset += 1
+
         self.peek_offset += 1
         return self.get(self.index + self.peek_offset)
 
@@ -58,7 +79,17 @@ def reset_peek(self, offset: int = 0) -> None:
 
     def skip_to_peek(self) -> None:
         self.index += self.peek_offset
+        self.row_index += self._peek_row_offset
+        if self._peek_row_offset:
+            # There have been newlines during the peek, so the column offset is the column index
+            # since the last newline
+            self.column_index = self._peek_column_offset
+        else:
+            self.column_index += self._peek_column_offset
+
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
 
 
 EOL = "\n"

fluent.syntax/tests/syntax/test_stream.py

@@ -159,3 +159,105 @@ def test_reset_peek(self):
 
         self.assertEqual("d", ps.peek())
         self.assertEqual(None, ps.peek())
+
+    def test_source_position_over_newline(self):
+        ps = ParserStream("ab\nc")
+
+        # Initially we should start at 0,0
+        self.assertEqual("a", ps.current_char)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        # Advancing within a line should increase the columm
+        self.assertEqual("b", ps.next())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        # Advancing to the newline should increase the column
+        self.assertEqual("\n", ps.next())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+        # Advancing past the newline should increase the row and reset the column
+        self.assertEqual("c", ps.next())
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+    def test_source_position_over_crlf(self):
+        ps = ParserStream("a\r\nb")
+
+        # Initially we should start at 0,0
+        self.assertEqual("a", ps.current_char)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        # Advancing to the CRLF should increase the column
+        self.assertEqual("\r", ps.next())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        # Advancing past the CRLF should increase the row and reset the column
+        self.assertEqual("b", ps.next())
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+    def test_skip_to_peek_over_newline(self):
+        ps = ParserStream("ab\nc")
+
+        # Initially we should start at 0,0
+        self.assertEqual("a", ps.current_char)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        # Peeking then advancing within a line should increase the columm
+        self.assertEqual("b", ps.peek())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        ps.skip_to_peek()
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        # Peeking then advancing to the newline should increase the column
+        self.assertEqual("\n", ps.peek())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        ps.skip_to_peek()
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+        # Peeking then advancing past the newline should increase the row and reset the column
+        self.assertEqual("c", ps.peek())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+        ps.skip_to_peek()
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+    def test_skip_to_peek_over_crlf(self):
+        ps = ParserStream("a\r\nb")
+
+        # Initially we should start at 0,0
+        self.assertEqual("a", ps.current_char)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        # Peeking then advancing to the CRLF should increase the column
+        self.assertEqual("\r", ps.peek())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        ps.skip_to_peek()
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        # Peeking then advancing past the CRLF should increase the row and reset the column
+        self.assertEqual("b", ps.peek())
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        ps.skip_to_peek()
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)