feat: add byte tracking

Cargo.toml

@@ -10,7 +10,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.39.0"
+version = "0.40.0"
 license = "MIT OR Apache-2.0"
 authors = [ "The html5ever Project Developers" ]
 repository = "https://github.com/servo/html5ever"
@@ -21,9 +21,9 @@ rust-version = "1.71.0"
 # Repo dependencies
 tendril = { version = "0.5", path = "tendril" }
 web_atoms = { version = "0.2.5", path = "web_atoms" }
-markup5ever = { version = "0.39", path = "markup5ever" }
-xml5ever = { version = "0.39", path = "xml5ever" }
-html5ever = { version = "0.39", path = "html5ever" }
+markup5ever = { version = "0.40", path = "markup5ever" }
+xml5ever = { version = "0.40", path = "xml5ever" }
+html5ever = { version = "0.40", path = "html5ever" }
 
 # External dependencies
 encoding_rs = "0.8.12"

html5ever/Cargo.toml

@@ -15,6 +15,7 @@ rust-version.workspace = true
 [features]
 trace_tokenizer = []
 serde = ["markup5ever/serde"]
+source-positions = ["markup5ever/source-positions"]
 
 [dependencies]
 markup5ever = { workspace = true }

html5ever/benches/html5ever.rs

@@ -8,14 +8,14 @@ use std::path::PathBuf;
 use criterion::{BatchSize, Criterion};
 
 use html5ever::tokenizer::{BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer};
-use html5ever::{tendril::*, TokenizerResult};
+use html5ever::{tendril::*, SourcePosition, TokenizerResult};
 
 struct Sink;
 
 impl TokenSink for Sink {
     type Handle = ();
 
-    fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+    fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> {
         // Don't use the token, but make sure we don't get
         // optimized out entirely.
         std::hint::black_box(token);

html5ever/examples/noop-tokenize.rs

@@ -16,6 +16,7 @@ use std::io;
 
 use html5ever::tendril::*;
 use html5ever::tokenizer::{BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer};
+use html5ever::SourcePosition;
 
 /// In our case, our sink only contains a tokens vector
 struct Sink(RefCell<Vec<Token>>);
@@ -24,7 +25,7 @@ impl TokenSink for Sink {
     type Handle = ();
 
     /// Each processed token will be handled by this method
-    fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+    fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> {
         self.0.borrow_mut().push(token);
         TokenSinkResult::Continue
     }

html5ever/examples/print-tree-actions.rs

@@ -20,7 +20,7 @@ use html5ever::tendril::*;
 use html5ever::tree_builder::{
     AppendNode, AppendText, ElementFlags, NodeOrText, QuirksMode, TreeSink,
 };
-use html5ever::{Attribute, QualName};
+use html5ever::{Attribute, QualName, SourcePosition};
 
 struct Sink {
     next_id: Cell<usize>,
@@ -160,8 +160,8 @@ impl TreeSink for Sink {
         println!("Mark script {node} as already started");
     }
 
-    fn set_current_line(&self, line_number: u64) {
-        println!("Set current line to {line_number}");
+    fn set_current_source_position(&self, position: SourcePosition) {
+        println!("Set current line to {}", position.line);
     }
 
     fn pop(&self, elem: &usize) {

html5ever/examples/tokenize.rs

@@ -18,6 +18,7 @@ use html5ever::tokenizer::{CharacterTokens, EndTag, NullCharacterToken, StartTag
 use html5ever::tokenizer::{
     ParseError, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
 };
+use html5ever::SourcePosition;
 
 #[derive(Clone)]
 struct TokenPrinter {
@@ -43,7 +44,7 @@ impl TokenPrinter {
 impl TokenSink for TokenPrinter {
     type Handle = ();
 
-    fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+    fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> {
         match token {
             CharacterTokens(b) => {
                 for c in b.chars() {

html5ever/src/tokenizer/char_ref/mod.rs

@@ -212,7 +212,11 @@ impl CharRefTokenizer {
             unconsume.push_char(c)
         }
 
+        #[cfg(feature = "source-positions")]
+        let unconsume_len = unconsume.len();
         input.push_front(unconsume);
+        #[cfg(feature = "source-positions")]
+        input.retreat_bytes_consumed(unconsume_len);
         tokenizer.emit_error(Borrowed("Numeric character reference without digits"));
         Status::Done(CharRef::EMPTY)
     }
@@ -292,7 +296,12 @@ impl CharRefTokenizer {
     }
 
     fn unconsume_name(&mut self, input: &BufferQueue) {
-        input.push_front(self.name_buf_opt.take().unwrap());
+        let name_buf = self.name_buf_opt.take().unwrap();
+        #[cfg(feature = "source-positions")]
+        let name_buf_len = name_buf.len();
+        input.push_front(name_buf);
+        #[cfg(feature = "source-positions")]
+        input.retreat_bytes_consumed(name_buf_len);
     }
 
     fn finish_named<Sink: TokenSink>(
@@ -367,7 +376,12 @@ impl CharRefTokenizer {
                     self.unconsume_name(input);
                     Status::Done(CharRef::EMPTY)
                 } else {
-                    input.push_front(StrTendril::from_slice(&self.name_buf()[name_len..]));
+                    let unconsumed = StrTendril::from_slice(&self.name_buf()[name_len..]);
+                    #[cfg(feature = "source-positions")]
+                    let unconsumed_len = unconsumed.len();
+                    input.push_front(unconsumed);
+                    #[cfg(feature = "source-positions")]
+                    input.retreat_bytes_consumed(unconsumed_len);
                     tokenizer.ignore_lf.set(false);
                     Status::Done(CharRef {
                         chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()],
@@ -419,6 +433,8 @@ impl CharRefTokenizer {
                 },
                 State::Octothorpe => {
                     input.push_front(StrTendril::from_slice("#"));
+                    #[cfg(feature = "source-positions")]
+                    input.retreat_bytes_consumed(1);
                     tokenizer.emit_error(Borrowed("EOF after '#' in character reference"));
                     Status::Done(CharRef::EMPTY)
                 },

html5ever/src/tokenizer/interface.rs

@@ -7,6 +7,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use markup5ever::SourcePosition;
 use markup5ever::ns;
 
 use crate::interface::Attribute;
@@ -125,7 +126,7 @@ pub trait TokenSink {
     type Handle;
 
     /// Process a token.
-    fn process_token(&self, token: Token, line_number: u64) -> TokenSinkResult<Self::Handle>;
+    fn process_token(&self, token: Token, position: SourcePosition) -> TokenSinkResult<Self::Handle>;
 
     /// Signal that tokenization reached the end of the document.
     fn end(&self) {}