Introduce using slices instead of locations

In the C API, we want to use slices instead of locations in the
AST. In this case a "slice" is effectively the same thing as the
location, expect it is represented using a 32-bit offset and a
32-bit length. This will cut down on half of the space of all of
the locations in the AST.

I am introducing this as a new field type to ease the migration
path, so that this can be merged on its own and then fields can
be moved one at a time.

Note that from the Ruby/Java/JavaScript side, this is effectively
an invisible change. This only impacts the C/Rust side.

Author: kddnewton

config.yml

@@ -819,7 +819,7 @@ nodes:
               alias $foo $bar
                          ^^^^
       - name: keyword_loc
-        type: location
+        type: slice
         comment: |
           The location of the `alias` keyword.
 

rbi/prism/reflection.rbi

@@ -35,6 +35,9 @@ end
 class Prism::Reflection::LocationField < Prism::Reflection::Field
 end
 
+class Prism::Reflection::SliceField < Prism::Reflection::Field
+end
+
 class Prism::Reflection::OptionalLocationField < Prism::Reflection::Field
 end
 

rust/ruby-prism/build.rs

@@ -31,6 +31,9 @@ enum NodeFieldType {
     #[serde(rename = "location")]
     Location,
 
+    #[serde(rename = "slice")]
+    Slice,
+
     #[serde(rename = "location?")]
     OptionalLocation,
 
@@ -369,6 +372,12 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box<d
                 writeln!(file, "        Location::new(self.parser, unsafe {{ &(*pointer) }})")?;
                 writeln!(file, "    }}")?;
             },
+            NodeFieldType::Slice => {
+                writeln!(file, "    pub fn {}(&self) -> Slice<'pr> {{", field.name)?;
+                writeln!(file, "        let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?;
+                writeln!(file, "        Slice::new(self.parser, unsafe {{ &(*pointer) }})")?;
+                writeln!(file, "    }}")?;
+            },
             NodeFieldType::OptionalLocation => {
                 writeln!(file, "    pub fn {}(&self) -> Option<Location<'pr>> {{", field.name)?;
                 writeln!(file, "        let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?;
@@ -558,7 +567,7 @@ use std::ptr::NonNull;
 
 #[allow(clippy::wildcard_imports)]
 use ruby_prism_sys::*;
-use crate::{{ConstantId, ConstantList, Integer, Location, NodeList}};
+use crate::{{ConstantId, ConstantList, Integer, Location, Slice, NodeList}};
 "
     )?;
 

rust/ruby-prism/src/lib.rs

@@ -19,7 +19,7 @@ use std::mem::MaybeUninit;
 use std::ptr::NonNull;
 
 pub use self::bindings::*;
-use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t};
+use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_slice_t};
 
 /// A range in the source file.
 pub struct Location<'pr> {
@@ -109,6 +109,53 @@ impl std::fmt::Debug for Location<'_> {
     }
 }
 
+/// A range in the source file, represented as a start offset and length.
+pub struct Slice<'pr> {
+    parser: NonNull<pm_parser_t>,
+    pub(crate) start: u32,
+    pub(crate) length: u32,
+    marker: PhantomData<&'pr [u8]>,
+}
+
+impl<'pr> Slice<'pr> {
+    /// Returns a byte slice for the range.
+    #[must_use]
+    pub fn as_slice(&self) -> &'pr [u8] {
+        unsafe {
+            let parser_start = (*self.parser.as_ptr()).start;
+            std::slice::from_raw_parts(parser_start.add(self.start as usize), self.length as usize)
+        }
+    }
+
+    /// Return a Slice from the given `pm_slice_t`.
+    #[must_use]
+    pub(crate) const fn new(parser: NonNull<pm_parser_t>, slice: &'pr pm_slice_t) -> Self {
+        Slice {
+            parser,
+            start: slice.start,
+            length: slice.length,
+            marker: PhantomData,
+        }
+    }
+}
+
+impl std::fmt::Debug for Slice<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let slice: &[u8] = self.as_slice();
+
+        let mut visible = String::new();
+        visible.push('"');
+
+        for &byte in slice {
+            let part: Vec<u8> = std::ascii::escape_default(byte).collect();
+            visible.push_str(std::str::from_utf8(&part).unwrap());
+        }
+
+        visible.push('"');
+        write!(f, "{visible}")
+    }
+}
+
 /// An iterator over the nodes in a list.
 pub struct NodeListIter<'pr> {
     parser: NonNull<pm_parser_t>,

sig/prism/reflection.rbs

@@ -30,6 +30,9 @@ module Prism
     class LocationField < Field
     end
 
+    class SliceField < Field
+    end
+
     class OptionalLocationField < Field
     end
 

src/prism.c

@@ -1947,6 +1947,8 @@ pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t
     return node;
 }
 
+#define TOKEN2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) })
+
 /**
  * Allocate and initialize a new AliasGlobalVariableNode node.
  */
@@ -1966,7 +1968,7 @@ pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyw
         },
         .new_name = new_name,
         .old_name = old_name,
-        .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword)
+        .keyword_loc = TOKEN2SLICE(parser, keyword)
     };
 
     return node;

templates/ext/prism/api_node.c.erb

@@ -27,6 +27,22 @@ pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *
     }
 }
 
+static VALUE
+pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) {
+    if (freeze) {
+        VALUE slice_argv[] = {
+            source,
+            LONG2FIX(start),
+            LONG2FIX(length)
+        };
+
+        return rb_obj_freeze(rb_class_new_instance(3, slice_argv, rb_cPrismLocation));
+    } else {
+        uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length));
+        return ULL2NUM(value);
+    }
+}
+
 VALUE
 pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) {
     ID type = rb_intern(pm_token_type_name(token->type));
@@ -241,6 +257,9 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     <%- when Prism::Template::OptionalLocationField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
                     argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
+                    <%- when Prism::Template::SliceField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze);
                     <%- when Prism::Template::UInt8Field -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
                     argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>);

templates/include/prism/ast.h.erb

@@ -57,6 +57,22 @@ typedef struct {
     const uint8_t *end;
 } pm_location_t;
 
+/**
+ * This struct represents a slice in the source code, defined by an offset and
+ * a length. Note that we have confirmation that we can represent all slices
+ * within Ruby source files using 32-bit integers per:
+ *
+ *     https://bugs.ruby-lang.org/issues/20488#note-1
+ *
+ */
+typedef struct {
+    /** The offset of the slice from the start of the source. */
+    uint32_t start;
+
+    /** The length of the slice. */
+    uint32_t length;
+} pm_slice_t;
+
 struct pm_node;
 
 /**
@@ -191,6 +207,7 @@ typedef struct pm_<%= node.human %> {
     when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}"
     when Prism::Template::StringField then "pm_string_t #{field.name}"
     when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_location_t #{field.name}"
+    when Prism::Template::SliceField then "pm_slice_t #{field.name}"
     when Prism::Template::UInt8Field then "uint8_t #{field.name}"
     when Prism::Template::UInt32Field then "uint32_t #{field.name}"
     when Prism::Template::IntegerField then "pm_integer_t #{field.name}"

templates/java/org/prism/Loader.java.erb

@@ -386,7 +386,7 @@ public class Loader {
               when Prism::Template::ConstantField then "loadConstant()"
               when Prism::Template::OptionalConstantField then "loadOptionalConstant()"
               when Prism::Template::ConstantListField then "loadConstants()"
-              when Prism::Template::LocationField then "loadLocation()"
+              when Prism::Template::LocationField, Prism::Template::SliceField then "loadLocation()"
               when Prism::Template::OptionalLocationField then "loadOptionalLocation()"
               when Prism::Template::UInt8Field then "buffer.get()"
               when Prism::Template::UInt32Field then "loadVarUInt()"

templates/java/org/prism/Nodes.java.erb

@@ -342,7 +342,7 @@ public abstract class Nodes {
             <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%>
             String nextNextIndent = nextIndent + "  ";
             <%- end -%>
-            <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%>
+            <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::SliceField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%>
             builder.append(nextIndent);
             builder.append("<%= field.name %>: ");
             <%- case field -%>