diff --git a/Cargo.toml b/Cargo.toml index 4714cdbc..4a4f4ba3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ ] [workspace.package] -version = "0.39.0" +version = "0.40.0" license = "MIT OR Apache-2.0" authors = [ "The html5ever Project Developers" ] repository = "https://github.com/servo/html5ever" @@ -21,9 +21,9 @@ rust-version = "1.71.0" # Repo dependencies tendril = { version = "0.5", path = "tendril" } web_atoms = { version = "0.2.5", path = "web_atoms" } -markup5ever = { version = "0.39", path = "markup5ever" } -xml5ever = { version = "0.39", path = "xml5ever" } -html5ever = { version = "0.39", path = "html5ever" } +markup5ever = { version = "0.40", path = "markup5ever" } +xml5ever = { version = "0.40", path = "xml5ever" } +html5ever = { version = "0.40", path = "html5ever" } # External dependencies encoding_rs = "0.8.12" diff --git a/html5ever/Cargo.toml b/html5ever/Cargo.toml index 3584e456..487168da 100644 --- a/html5ever/Cargo.toml +++ b/html5ever/Cargo.toml @@ -15,6 +15,7 @@ rust-version.workspace = true [features] trace_tokenizer = [] serde = ["markup5ever/serde"] +source-positions = ["markup5ever/source-positions"] [dependencies] markup5ever = { workspace = true } diff --git a/html5ever/benches/html5ever.rs b/html5ever/benches/html5ever.rs index f74b13a7..1af8a12d 100644 --- a/html5ever/benches/html5ever.rs +++ b/html5ever/benches/html5ever.rs @@ -8,14 +8,14 @@ use std::path::PathBuf; use criterion::{BatchSize, Criterion}; use html5ever::tokenizer::{BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer}; -use html5ever::{tendril::*, TokenizerResult}; +use html5ever::{tendril::*, SourcePosition, TokenizerResult}; struct Sink; impl TokenSink for Sink { type Handle = (); - fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> { + fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> { // Don't use the token, but make sure we don't get // optimized out entirely. std::hint::black_box(token); diff --git a/html5ever/examples/noop-tokenize.rs b/html5ever/examples/noop-tokenize.rs index a95404df..9f5df614 100644 --- a/html5ever/examples/noop-tokenize.rs +++ b/html5ever/examples/noop-tokenize.rs @@ -16,6 +16,7 @@ use std::io; use html5ever::tendril::*; use html5ever::tokenizer::{BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer}; +use html5ever::SourcePosition; /// In our case, our sink only contains a tokens vector struct Sink(RefCell>); @@ -24,7 +25,7 @@ impl TokenSink for Sink { type Handle = (); /// Each processed token will be handled by this method - fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> { + fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> { self.0.borrow_mut().push(token); TokenSinkResult::Continue } diff --git a/html5ever/examples/print-tree-actions.rs b/html5ever/examples/print-tree-actions.rs index dfa0aedd..14044454 100644 --- a/html5ever/examples/print-tree-actions.rs +++ b/html5ever/examples/print-tree-actions.rs @@ -20,7 +20,7 @@ use html5ever::tendril::*; use html5ever::tree_builder::{ AppendNode, AppendText, ElementFlags, NodeOrText, QuirksMode, TreeSink, }; -use html5ever::{Attribute, QualName}; +use html5ever::{Attribute, QualName, SourcePosition}; struct Sink { next_id: Cell, @@ -160,8 +160,8 @@ impl TreeSink for Sink { println!("Mark script {node} as already started"); } - fn set_current_line(&self, line_number: u64) { - println!("Set current line to {line_number}"); + fn set_current_source_position(&self, position: SourcePosition) { + println!("Set current line to {}", position.line); } fn pop(&self, elem: &usize) { diff --git a/html5ever/examples/tokenize.rs b/html5ever/examples/tokenize.rs index ba984d8f..3e5a5f35 100644 --- a/html5ever/examples/tokenize.rs +++ b/html5ever/examples/tokenize.rs @@ -18,6 +18,7 @@ use html5ever::tokenizer::{CharacterTokens, EndTag, NullCharacterToken, StartTag use html5ever::tokenizer::{ ParseError, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts, }; +use html5ever::SourcePosition; #[derive(Clone)] struct TokenPrinter { @@ -43,7 +44,7 @@ impl TokenPrinter { impl TokenSink for TokenPrinter { type Handle = (); - fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> { + fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> { match token { CharacterTokens(b) => { for c in b.chars() { diff --git a/html5ever/src/tokenizer/char_ref/mod.rs b/html5ever/src/tokenizer/char_ref/mod.rs index e119477d..9052ae6b 100644 --- a/html5ever/src/tokenizer/char_ref/mod.rs +++ b/html5ever/src/tokenizer/char_ref/mod.rs @@ -212,7 +212,11 @@ impl CharRefTokenizer { unconsume.push_char(c) } + #[cfg(feature = "source-positions")] + let unconsume_len = unconsume.len(); input.push_front(unconsume); + #[cfg(feature = "source-positions")] + input.retreat_bytes_consumed(unconsume_len); tokenizer.emit_error(Borrowed("Numeric character reference without digits")); Status::Done(CharRef::EMPTY) } @@ -292,7 +296,12 @@ impl CharRefTokenizer { } fn unconsume_name(&mut self, input: &BufferQueue) { - input.push_front(self.name_buf_opt.take().unwrap()); + let name_buf = self.name_buf_opt.take().unwrap(); + #[cfg(feature = "source-positions")] + let name_buf_len = name_buf.len(); + input.push_front(name_buf); + #[cfg(feature = "source-positions")] + input.retreat_bytes_consumed(name_buf_len); } fn finish_named( @@ -367,7 +376,12 @@ impl CharRefTokenizer { self.unconsume_name(input); Status::Done(CharRef::EMPTY) } else { - input.push_front(StrTendril::from_slice(&self.name_buf()[name_len..])); + let unconsumed = StrTendril::from_slice(&self.name_buf()[name_len..]); + #[cfg(feature = "source-positions")] + let unconsumed_len = unconsumed.len(); + input.push_front(unconsumed); + #[cfg(feature = "source-positions")] + input.retreat_bytes_consumed(unconsumed_len); tokenizer.ignore_lf.set(false); Status::Done(CharRef { chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()], @@ -419,6 +433,8 @@ impl CharRefTokenizer { }, State::Octothorpe => { input.push_front(StrTendril::from_slice("#")); + #[cfg(feature = "source-positions")] + input.retreat_bytes_consumed(1); tokenizer.emit_error(Borrowed("EOF after '#' in character reference")); Status::Done(CharRef::EMPTY) }, diff --git a/html5ever/src/tokenizer/interface.rs b/html5ever/src/tokenizer/interface.rs index b1436a71..d2a0609b 100644 --- a/html5ever/src/tokenizer/interface.rs +++ b/html5ever/src/tokenizer/interface.rs @@ -7,6 +7,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +use markup5ever::SourcePosition; use markup5ever::ns; use crate::interface::Attribute; @@ -125,7 +126,7 @@ pub trait TokenSink { type Handle; /// Process a token. - fn process_token(&self, token: Token, line_number: u64) -> TokenSinkResult; + fn process_token(&self, token: Token, position: SourcePosition) -> TokenSinkResult; /// Signal that tokenization reached the end of the document. fn end(&self) {} diff --git a/html5ever/src/tokenizer/mod.rs b/html5ever/src/tokenizer/mod.rs index dcd372da..a883f959 100644 --- a/html5ever/src/tokenizer/mod.rs +++ b/html5ever/src/tokenizer/mod.rs @@ -25,7 +25,7 @@ use self::char_ref::{CharRef, CharRefTokenizer}; use crate::util::str::lower_ascii_letter; use log::{debug, trace}; -use markup5ever::{ns, small_char_set, TokenizerResult}; +use markup5ever::{ns, small_char_set, SourcePosition, TokenizerResult}; use std::borrow::Cow::{self, Borrowed}; use std::cell::{Cell, RefCell, RefMut}; use std::cmp::Reverse; @@ -181,6 +181,30 @@ pub struct Tokenizer { /// Track current line current_line: Cell, + + /// Number of UTF-8 bytes consumed from the input so far. + /// + /// Kept in sync with `BufferQueue::bytes_consumed` after every character + /// is consumed. + #[cfg(feature = "source-positions")] + current_byte: Cell, + + /// Byte offset of the first character of the current token. + /// + /// For tag, comment, and doctype tokens this is the byte of the `<` that + /// opened them, captured whenever `<` is consumed in `get_preprocessed_char`. + /// + /// For character tokens it is the byte right after the end of the previous token, + /// which equals the first byte of the text content, this is tracked via `last_token_end_byte`. + #[cfg(feature = "source-positions")] + token_start_byte: Cell, + + /// Byte offset one past the end of the most recently emitted token. + /// + /// Updated at the end of each `process_token` call. Used as the start + /// byte for the next character token. + #[cfg(feature = "source-positions")] + last_token_end_byte: Cell, } impl Tokenizer { @@ -216,6 +240,12 @@ impl Tokenizer { state_profile: RefCell::new(BTreeMap::new()), time_in_sink: Cell::new(0), current_line: Cell::new(1), + #[cfg(feature = "source-positions")] + current_byte: Cell::new(0), + #[cfg(feature = "source-positions")] + token_start_byte: Cell::new(0), + #[cfg(feature = "source-positions")] + last_token_end_byte: Cell::new(0), } } @@ -242,14 +272,39 @@ impl Tokenizer { self.state.set(states::Plaintext); } + fn token_byte(&self, token: &Token) -> Option { + #[cfg(feature = "source-positions")] + { + Some(match token { + Token::TagToken(_) | Token::CommentToken(_) | Token::DoctypeToken(_) => { + self.token_start_byte.get() + }, + Token::CharacterTokens(_) => self.last_token_end_byte.get(), + _ => self.current_byte.get(), + }) + } + #[cfg(not(feature = "source-positions"))] + { + let _ = token; + None + } + } + fn process_token(&self, token: Token) -> TokenSinkResult { - if self.opts.profile { - let (ret, dt) = time!(self.sink.process_token(token, self.current_line.get())); + let position = SourcePosition { + line: self.current_line.get(), + byte: self.token_byte(&token), + }; + let result = if self.opts.profile { + let (ret, dt) = time!(self.sink.process_token(token, position)); self.time_in_sink.set(self.time_in_sink.get() + dt); ret } else { - self.sink.process_token(token, self.current_line.get()) - } + self.sink.process_token(token, position) + }; + #[cfg(feature = "source-positions")] + self.last_token_end_byte.set(self.current_byte.get()); + result } fn process_token_and_continue(&self, token: Token) { @@ -292,6 +347,17 @@ impl Tokenizer { trace!("got character {c}"); self.current_char.set(c); + #[cfg(feature = "source-positions")] + { + let pos = input.bytes_consumed(); + if pos > 0 { + self.current_byte.set(pos); + if c == '<' { + self.token_start_byte + .set(pos.saturating_sub(c.len_utf8())); + } + } + } Some(c) } @@ -325,7 +391,13 @@ impl Tokenizer { // NB: We don't set self.current_char for a run of characters not // in the set. It shouldn't matter for the codepaths that use // this. - _ => d, + other => { + #[cfg(feature = "source-positions")] + if other.is_some() { + self.current_byte.set(input.bytes_consumed()); + } + other + }, } } @@ -618,7 +690,20 @@ impl Tokenizer { if self.reconsume.get() { self.reconsume.set(false); } else { + #[cfg(not(feature = "source-positions"))] input.next(); + #[cfg(feature = "source-positions")] + { + let c = input.next(); + if let Some(c) = c { + let pos = input.bytes_consumed(); + self.current_byte.set(pos); + if c == '<' { + self.token_start_byte + .set(pos.saturating_sub(c.len_utf8())); + } + } + } } } @@ -752,14 +837,29 @@ impl Tokenizer { } else { // SAFETY: // This CPU is guaranteed to support SIMD due to the is_supported_simd_feature_detected check above - let result = unsafe { self.data_state_simd_fast_path(&mut front_buffer) }; + let simd_result = + unsafe { self.data_state_simd_fast_path(&mut front_buffer) }; + + #[cfg(feature = "source-positions")] + if let Some(ref result) = simd_result { + let consumed_bytes = match result { + SetResult::NotFromSet(ref text) => text.len(), + SetResult::FromSet(character) => character.len_utf8(), + }; + input.advance_bytes_consumed(consumed_bytes); + self.current_byte.set(input.bytes_consumed()); + if let SetResult::FromSet('<') = result { + self.token_start_byte + .set(input.bytes_consumed() - '<'.len_utf8()); + } + } if front_buffer.is_empty() { drop(front_buffer); input.pop_front(); } - result + simd_result } } else { self.pop_except_from(input, set) @@ -1749,6 +1849,8 @@ impl Tokenizer { let mut char_ref_tokenizer = self.char_ref_tokenizer.borrow_mut(); let progress = match char_ref_tokenizer.as_mut().unwrap().step(self, input) { char_ref::Status::Done(char_ref) => { + #[cfg(feature = "source-positions")] + self.current_byte.set(input.bytes_consumed()); self.process_char_ref(char_ref); *char_ref_tokenizer = None; return ProcessResult::Continue; @@ -2204,6 +2306,7 @@ mod test { use super::interface::{TagToken, Token}; use markup5ever::buffer_queue::BufferQueue; + use markup5ever::SourcePosition; use std::cell::RefCell; use crate::LocalName; @@ -2242,7 +2345,7 @@ mod test { impl TokenSink for LinesMatch { type Handle = (); - fn process_token(&self, token: Token, line_number: u64) -> TokenSinkResult { + fn process_token(&self, token: Token, position: SourcePosition) -> TokenSinkResult { match token { CharacterTokens(b) => { self.current_str.borrow_mut().push_slice(&b); @@ -2267,12 +2370,12 @@ mod test { }, _ => t.attrs.sort_by(|a1, a2| a1.name.cmp(&a2.name)), } - self.push(TagToken(t), line_number); + self.push(TagToken(t), position.line); }, EOFToken => (), - _ => self.push(token, line_number), + _ => self.push(token, position.line), } TokenSinkResult::Continue } @@ -2375,4 +2478,213 @@ mod test { let results = tokenize(vector, opts); assert_eq!(results, expected); } + + #[cfg(feature = "source-positions")] + mod test_source_positions { + use super::super::interface::{CharacterTokens, EOFToken, NullCharacterToken, TagToken}; + use super::super::interface::{EndTag, StartTag, Tag, Token}; + use super::super::{TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts}; + + use crate::tendril::StrTendril; + use crate::LocalName; + use markup5ever::buffer_queue::BufferQueue; + use markup5ever::SourcePosition; + use std::cell::RefCell; + + struct BytesMatch { + text_start_byte: std::cell::Cell>, + current_str: RefCell, + entries: RefCell>, + } + + impl BytesMatch { + fn new() -> Self { + BytesMatch { + text_start_byte: std::cell::Cell::new(None), + current_str: RefCell::new(StrTendril::new()), + entries: RefCell::new(vec![]), + } + } + + fn flush_chars(&self) { + let s = self.current_str.take(); + if !s.is_empty() { + let byte = self.text_start_byte.get().unwrap_or(0); + self.text_start_byte.set(None); + self.entries.borrow_mut().push((CharacterTokens(s), byte)); + } + } + } + + struct RawBytesMatch { + entries: RefCell>, + } + + impl RawBytesMatch { + fn new() -> Self { + RawBytesMatch { + entries: RefCell::new(vec![]), + } + } + } + + impl TokenSink for RawBytesMatch { + type Handle = (); + + fn process_token( + &self, + token: Token, + position: SourcePosition, + ) -> TokenSinkResult { + if !matches!(token, EOFToken) { + self.entries + .borrow_mut() + .push((token, position.byte.unwrap_or(0))); + } + TokenSinkResult::Continue + } + } + + impl TokenSink for BytesMatch { + type Handle = (); + + fn process_token( + &self, + token: Token, + position: SourcePosition, + ) -> TokenSinkResult { + let byte = position.byte.unwrap_or(0); + match token { + CharacterTokens(b) => { + if self.text_start_byte.get().is_none() { + self.text_start_byte.set(Some(byte)); + } + self.current_str.borrow_mut().push_slice(&b); + }, + NullCharacterToken => { + self.current_str.borrow_mut().push_char('\0'); + }, + EOFToken => { + self.flush_chars(); + }, + TagToken(t) => { + self.flush_chars(); + self.entries.borrow_mut().push((TagToken(t), byte)); + }, + other => { + self.flush_chars(); + self.entries.borrow_mut().push((other, byte)); + }, + } + TokenSinkResult::Continue + } + } + + fn tokenize_bytes(input: &str) -> Vec<(Token, usize)> { + let sink = BytesMatch::new(); + let tok = Tokenizer::new(sink, TokenizerOpts::default()); + let buf = BufferQueue::default(); + buf.push_back(StrTendril::from(input)); + let _ = tok.feed(&buf); + tok.end(); + tok.sink.entries.take() + } + + fn tokenize_raw_bytes(input: &str) -> Vec<(Token, usize)> { + let sink = RawBytesMatch::new(); + let tok = Tokenizer::new(sink, TokenizerOpts::default()); + let buf = BufferQueue::default(); + buf.push_back(StrTendril::from(input)); + let _ = tok.feed(&buf); + tok.end(); + tok.sink.entries.take() + } + + fn start(name: &str) -> Token { + TagToken(Tag { + kind: StartTag, + name: LocalName::from(name), + self_closing: false, + attrs: vec![], + had_duplicate_attributes: false, + }) + } + + fn end(name: &str) -> Token { + TagToken(Tag { + kind: EndTag, + name: LocalName::from(name), + self_closing: false, + attrs: vec![], + had_duplicate_attributes: false, + }) + } + + fn chars(s: &str) -> Token { + CharacterTokens(StrTendril::from(s)) + } + + #[test] + fn check_byte_offsets_simple_tags() { + let entries = tokenize_bytes(""); + assert_eq!( + entries, + vec![ + (start("a"), 0), + (start("b"), 3), + (end("b"), 6), + (end("a"), 10), + ] + ); + } + + #[test] + fn check_byte_offsets_text_content() { + let entries = tokenize_bytes("

hello

"); + assert_eq!( + entries, + vec![(start("p"), 0), (chars("hello"), 3), (end("p"), 8),] + ); + } + + #[test] + fn check_byte_offsets_multibyte_text() { + let entries = tokenize_bytes("

é

"); + assert_eq!( + entries, + vec![(start("p"), 0), (chars("é"), 3), (end("p"), 5),] + ); + } + + #[test] + fn check_byte_offsets_sequential_siblings() { + let entries = tokenize_bytes("

X

Y

"); + assert_eq!( + entries, + vec![ + (start("h1"), 0), + (chars("X"), 4), + (end("h1"), 5), + (start("p"), 10), + (chars("Y"), 13), + (end("p"), 14), + ] + ); + } + + #[test] + fn check_byte_offsets_entity_text_chunks() { + let entries = tokenize_raw_bytes("

a&b

"); + assert_eq!( + entries, + vec![ + (start("p"), 0), + (chars("a"), 3), + (chars("&"), 4), + (chars("b"), 9), + (end("p"), 10), + ] + ); + } + } } diff --git a/html5ever/src/tree_builder/mod.rs b/html5ever/src/tree_builder/mod.rs index 3fcfaec3..80c60e9f 100644 --- a/html5ever/src/tree_builder/mod.rs +++ b/html5ever/src/tree_builder/mod.rs @@ -474,9 +474,13 @@ where { type Handle = Handle; - fn process_token(&self, token: tokenizer::Token, line_number: u64) -> TokenSinkResult { - if line_number != self.current_line.get() { - self.sink.set_current_line(line_number); + fn process_token( + &self, + token: tokenizer::Token, + position: markup5ever::SourcePosition, + ) -> TokenSinkResult { + if position.line != self.current_line.get() || position.byte.is_some() { + self.sink.set_current_source_position(position); } let ignore_lf = self.ignore_lf.take(); diff --git a/markup5ever/Cargo.toml b/markup5ever/Cargo.toml index 764f9a0a..385fcdc6 100644 --- a/markup5ever/Cargo.toml +++ b/markup5ever/Cargo.toml @@ -15,6 +15,7 @@ path = "lib.rs" [features] serde = ["web_atoms/serde"] +source-positions = [] [dependencies] web_atoms = { workspace = true } diff --git a/markup5ever/interface/mod.rs b/markup5ever/interface/mod.rs index 247daf35..e4c89672 100644 --- a/markup5ever/interface/mod.rs +++ b/markup5ever/interface/mod.rs @@ -17,6 +17,17 @@ pub use self::tree_builder::{create_element, AppendNode, AppendText, ElementFlag pub use self::tree_builder::{ElemName, Tracer, TreeSink}; pub use self::tree_builder::{LimitedQuirks, NoQuirks, Quirks, QuirksMode}; +/// A position in the input stream passed alongside each token. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Default)] +pub struct SourcePosition { + /// 1-based line number in the input. + pub line: u64, + /// UTF-8 byte offset of the token start in the original input. + /// + /// `None` when the `source-positions` feature is disabled. + pub byte: Option, +} + /// An [expanded name], containing the tag and the namespace. /// /// [expanded name]: https://www.w3.org/TR/REC-xml-names/#dt-expname diff --git a/markup5ever/interface/tree_builder.rs b/markup5ever/interface/tree_builder.rs index e1683de0..36e9f6dd 100644 --- a/markup5ever/interface/tree_builder.rs +++ b/markup5ever/interface/tree_builder.rs @@ -266,8 +266,8 @@ pub trait TreeSink { false } - /// Called whenever the line number changes. - fn set_current_line(&self, _line_number: u64) {} + /// Called whenever the source position changes before a tree-builder callback. + fn set_current_source_position(&self, _position: super::SourcePosition) {} fn allow_declarative_shadow_roots(&self, _intended_parent: &Self::Handle) -> bool { true diff --git a/markup5ever/lib.rs b/markup5ever/lib.rs index a591fec4..e6b954f3 100644 --- a/markup5ever/lib.rs +++ b/markup5ever/lib.rs @@ -50,6 +50,6 @@ mod util { pub mod smallcharset; } -pub use interface::{Attribute, ExpandedName, QualName, TokenizerResult}; +pub use interface::{Attribute, ExpandedName, QualName, SourcePosition, TokenizerResult}; pub use util::smallcharset::SmallCharSet; pub use util::*; diff --git a/markup5ever/util/buffer_queue.rs b/markup5ever/util/buffer_queue.rs index d5e6864f..ba768b75 100644 --- a/markup5ever/util/buffer_queue.rs +++ b/markup5ever/util/buffer_queue.rs @@ -18,6 +18,8 @@ //! //! [`BufferQueue`]: struct.BufferQueue.html +#[cfg(feature = "source-positions")] +use std::cell::Cell; use std::{ cell::{RefCell, RefMut}, collections::VecDeque, @@ -51,6 +53,12 @@ pub enum SetResult { pub struct BufferQueue { /// Buffers to process. buffers: RefCell>, + /// Total number of UTF-8 bytes consumed from this queue so far. + /// + /// Used by the tokenizer to surface byte-accurate source offsets via + /// [`SourcePosition`]. + #[cfg(feature = "source-positions")] + bytes_consumed: Cell, } impl Default for BufferQueue { @@ -59,6 +67,8 @@ impl Default for BufferQueue { fn default() -> Self { Self { buffers: RefCell::new(VecDeque::with_capacity(16)), + #[cfg(feature = "source-positions")] + bytes_consumed: Cell::new(0), } } } @@ -70,6 +80,39 @@ impl BufferQueue { self.buffers.borrow().is_empty() } + /// Returns the total number of UTF-8 bytes consumed from this queue. + /// + /// The value monotonically increases as characters are consumed via + /// [`next`], [`pop_except_from`], and [`eat`]. Re-queuing bytes via + /// [`push_front`] does **not** decrement the counter. + /// + /// To reduce bytes_consumed, use [`retreat_bytes_consumed`]. + #[cfg(feature = "source-positions")] + #[inline] + pub fn bytes_consumed(&self) -> usize { + self.bytes_consumed.get() + } + + /// Advance the bytes-consumed counter by `n`. + /// + /// Use this to manually advance the counter when bypassing: [`next`], [`pop_except_from`], and [`eat`] + #[cfg(feature = "source-positions")] + #[inline] + pub fn advance_bytes_consumed(&self, n: usize) { + self.bytes_consumed.set(self.bytes_consumed.get() + n); + } + + /// Retreat the bytes-consumed counter by `n`. + /// + /// Used by tokenizer lookahead paths that consume raw bytes, then push unmatched + /// suffix bytes back onto the queue. + #[cfg(feature = "source-positions")] + #[inline] + pub fn retreat_bytes_consumed(&self, n: usize) { + self.bytes_consumed + .set(self.bytes_consumed.get().saturating_sub(n)); + } + /// Get the buffer at the beginning of the queue. #[inline] pub fn pop_front(&self) -> Option { @@ -146,9 +189,15 @@ impl BufferQueue { out = buf.unsafe_subtendril(0, n); buf.unsafe_pop_front(n); } + #[cfg(feature = "source-positions")] + self.bytes_consumed + .set(self.bytes_consumed.get() + out.len()); (Some(NotFromSet(out)), buf.is_empty()) } else { let c = buf.pop_front_char().expect("empty buffer in queue"); + #[cfg(feature = "source-positions")] + self.bytes_consumed + .set(self.bytes_consumed.get() + c.len_utf8()); (Some(FromSet(c)), buf.is_empty()) } }, @@ -218,6 +267,10 @@ impl BufferQueue { Some(ref mut buf) => buf.pop_front(consumed_from_last as u32), } + #[cfg(feature = "source-positions")] + self.bytes_consumed + .set(self.bytes_consumed.get() + pat.len()); + Some(true) } @@ -229,6 +282,9 @@ impl BufferQueue { None => (None, false), Some(buf) => { let c = buf.pop_front_char().expect("empty buffer in queue"); + #[cfg(feature = "source-positions")] + self.bytes_consumed + .set(self.bytes_consumed.get() + c.len_utf8()); (Some(c), buf.is_empty()) }, }; @@ -331,3 +387,147 @@ mod test { assert_eq!(bq.next(), None); } } + +#[cfg(all(test, feature = "source-positions"))] +mod test_source_positions { + use tendril::SliceExt; + + use super::BufferQueue; + use super::SetResult::{FromSet, NotFromSet}; + + #[test] + fn next_advances_counter_by_utf8_width_single() { + let bq = BufferQueue::default(); + assert_eq!(bq.bytes_consumed(), 0); + + bq.push_back("abc".to_tendril()); + bq.next(); + assert_eq!(bq.bytes_consumed(), 1); + bq.next(); + assert_eq!(bq.bytes_consumed(), 2); + bq.next(); + assert_eq!(bq.bytes_consumed(), 3); + } + + #[test] + fn next_advances_counter_by_utf8_width_double() { + let bq = BufferQueue::default(); + assert_eq!(bq.bytes_consumed(), 0); + + bq.push_back("é".to_tendril()); + bq.next(); + assert_eq!(bq.bytes_consumed(), 2); + } + + #[test] + fn pop_except_from_not_from_set_advances_counter() { + let bq = BufferQueue::default(); + bq.push_back("abc&".to_tendril()); + let set = small_char_set!('&'); + + assert_eq!( + bq.pop_except_from(set), + Some(NotFromSet("abc".to_tendril())) + ); + assert_eq!(bq.bytes_consumed(), 3); + } + + #[test] + fn pop_except_from_from_set_advances_counter() { + let bq = BufferQueue::default(); + bq.push_back("&def".to_tendril()); + let set = small_char_set!('&'); + + assert_eq!(bq.pop_except_from(set), Some(FromSet('&'))); + assert_eq!(bq.bytes_consumed(), 1); + } + + #[test] + fn pop_except_from_successive_calls_accumulate_counter() { + let bq = BufferQueue::default(); + bq.push_back("abc&def".to_tendril()); + let set = small_char_set!('&'); + + bq.pop_except_from(set); + assert_eq!(bq.bytes_consumed(), 3); + + bq.pop_except_from(set); + assert_eq!(bq.bytes_consumed(), 4); + + bq.pop_except_from(set); + assert_eq!(bq.bytes_consumed(), 7); + } + + #[test] + fn pop_except_from_multibyte_bulk_advances_by_byte_len() { + let bq = BufferQueue::default(); + bq.push_back("café&".to_tendril()); + let set = small_char_set!('&'); + + let result = bq.pop_except_from(set); + assert!(matches!(result, Some(NotFromSet(_)))); + assert_eq!(bq.bytes_consumed(), 5); + } + + #[test] + fn eat_advances_counter_accordingly() { + let bq = BufferQueue::default(); + bq.push_back("abcdef".to_tendril()); + + assert_eq!(bq.eat("ax", u8::eq_ignore_ascii_case), Some(false)); + assert_eq!(bq.bytes_consumed(), 0); + + assert_eq!(bq.eat("abc", u8::eq_ignore_ascii_case), Some(true)); + assert_eq!(bq.bytes_consumed(), 3); + + assert_eq!(bq.eat("def", u8::eq_ignore_ascii_case), Some(true)); + assert_eq!(bq.bytes_consumed(), 6); + } + + #[test] + /// This test is to ensure the behaviour contract of push_front is kept. + /// There are use cases where pushing front should technically not retreat the + /// bytes counter, so it's up to the caller to decide if pushing front should retreat. + fn push_front_does_not_decrement_counter() { + let bq = BufferQueue::default(); + bq.push_back("abc".to_tendril()); + bq.next(); + bq.next(); + assert_eq!(bq.bytes_consumed(), 2); + + bq.push_front("xy".to_tendril()); + assert_eq!(bq.bytes_consumed(), 2); + + bq.next(); + bq.next(); + assert_eq!(bq.bytes_consumed(), 4); + } + + #[test] + fn advance_bytes_consumed_adds_exactly() { + let bq = BufferQueue::default(); + assert_eq!(bq.bytes_consumed(), 0); + + bq.advance_bytes_consumed(7); + assert_eq!(bq.bytes_consumed(), 7); + + bq.advance_bytes_consumed(3); + assert_eq!(bq.bytes_consumed(), 10); + } + + #[test] + fn retreat_bytes_consumed_subtracts_exactly() { + let bq = BufferQueue::default(); + bq.advance_bytes_consumed(10); + assert_eq!(bq.bytes_consumed(), 10); + + bq.retreat_bytes_consumed(3); + assert_eq!(bq.bytes_consumed(), 7); + + bq.retreat_bytes_consumed(7); + assert_eq!(bq.bytes_consumed(), 0); + + bq.retreat_bytes_consumed(5); + assert_eq!(bq.bytes_consumed(), 0); + } +} diff --git a/rcdom/Cargo.toml b/rcdom/Cargo.toml index caf52b54..f402b541 100644 --- a/rcdom/Cargo.toml +++ b/rcdom/Cargo.toml @@ -20,6 +20,9 @@ markup5ever = { workspace = true, features = ["serde"] } tendril = { workspace = true } xml5ever = { workspace = true } +[features] +source-positions = ["html5ever/source-positions"] + [dev-dependencies] criterion = { workspace = true } env_logger = { workspace = true } @@ -45,3 +48,7 @@ harness = false [[test]] name = "xml-tokenizer" harness = false + +[[test]] +name = "source-positions-integration" +required-features = ["source-positions"] diff --git a/rcdom/tests/duplicate-attributes-integration.rs b/rcdom/tests/duplicate-attributes-integration.rs index 72164ba4..e9037550 100644 --- a/rcdom/tests/duplicate-attributes-integration.rs +++ b/rcdom/tests/duplicate-attributes-integration.rs @@ -12,7 +12,7 @@ use html5ever::tendril::stream::TendrilSink; use html5ever::tendril::StrTendril; use html5ever::ExpandedName; use html5ever::QualName; -use markup5ever::interface::{ElementFlags, NodeOrText, QuirksMode, TreeSink}; +use markup5ever::interface::{ElementFlags, NodeOrText, QuirksMode, SourcePosition, TreeSink}; use markup5ever::{local_name, ns, Attribute}; use markup5ever_rcdom::{Handle, RcDom}; use std::borrow::Cow; @@ -128,8 +128,8 @@ impl TreeSink for FlagCapturingDOM { self.rcdom.mark_script_already_started(node) } - fn set_current_line(&self, line_number: u64) { - self.rcdom.set_current_line(line_number) + fn set_current_source_position(&self, position: SourcePosition) { + TreeSink::set_current_source_position(&self.rcdom, position); } fn pop(&self, node: &Handle) { diff --git a/rcdom/tests/html-serializer.rs b/rcdom/tests/html-serializer.rs index 67e3ca79..dfbbaae4 100644 --- a/rcdom/tests/html-serializer.rs +++ b/rcdom/tests/html-serializer.rs @@ -11,7 +11,7 @@ use html5ever::driver::ParseOpts; use html5ever::serialize::{Serialize, SerializeOpts, Serializer, TraversalScope}; use html5ever::tendril::{SliceExt, StrTendril, TendrilSink}; use html5ever::tokenizer::{TagKind, Token, TokenSink, TokenSinkResult, Tokenizer}; -use html5ever::{parse_document, parse_fragment, serialize, QualName}; +use html5ever::{parse_document, parse_fragment, serialize, QualName, SourcePosition}; use markup5ever::{local_name, ns}; use markup5ever_rcdom::{RcDom, SerializableHandle}; @@ -23,7 +23,7 @@ struct Tokens(RefCell>); impl TokenSink for Tokens { type Handle = (); - fn process_token(&self, token: Token, _: u64) -> TokenSinkResult<()> { + fn process_token(&self, token: Token, _: SourcePosition) -> TokenSinkResult<()> { self.0.borrow_mut().push(token); TokenSinkResult::Continue } diff --git a/rcdom/tests/html-tokenizer.rs b/rcdom/tests/html-tokenizer.rs index 7ac8767a..f50edc0b 100644 --- a/rcdom/tests/html-tokenizer.rs +++ b/rcdom/tests/html-tokenizer.rs @@ -20,7 +20,7 @@ use html5ever::tokenizer::{CommentToken, DoctypeToken, TagToken, Token}; use html5ever::tokenizer::{Doctype, EndTag, StartTag, Tag}; use html5ever::tokenizer::{TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts}; use html5ever::TokenizerResult; -use html5ever::{ns, Attribute, LocalName, QualName}; +use html5ever::{ns, Attribute, LocalName, QualName, SourcePosition}; use serde_json::{Map, Value}; use std::cell::RefCell; use std::char; @@ -108,7 +108,7 @@ impl TokenLogger { impl TokenSink for TokenLogger { type Handle = (); - fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> { + fn process_token(&self, token: Token, _position: SourcePosition) -> TokenSinkResult<()> { match token { CharacterTokens(b) => { self.current_str.borrow_mut().push_slice(&b); @@ -283,14 +283,14 @@ fn json_to_tokens( let sink = TokenLogger::new(exact_errors); for tok in js_tokens.get_list().iter() { assert_eq!( - sink.process_token(json_to_token(tok), 0), + sink.process_token(json_to_token(tok), SourcePosition::default()), TokenSinkResult::Continue ); } for err in js_errors { assert_eq!( - sink.process_token(ParseError(err.find("code").get_str().into()), 0), + sink.process_token(ParseError(err.find("code").get_str().into()), SourcePosition::default()), TokenSinkResult::Continue ); } diff --git a/rcdom/tests/html-tree-sink.rs b/rcdom/tests/html-tree-sink.rs index c5371593..b222ae5e 100644 --- a/rcdom/tests/html-tree-sink.rs +++ b/rcdom/tests/html-tree-sink.rs @@ -3,7 +3,7 @@ use html5ever::tendril::stream::TendrilSink; use html5ever::tendril::StrTendril; use html5ever::ExpandedName; use html5ever::QualName; -use markup5ever::interface::{ElementFlags, NodeOrText, QuirksMode, TreeSink}; +use markup5ever::interface::{ElementFlags, NodeOrText, QuirksMode, SourcePosition, TreeSink}; use markup5ever::{local_name, ns, Attribute}; use markup5ever_rcdom::{Handle, RcDom}; use std::borrow::Cow; @@ -108,8 +108,8 @@ impl TreeSink for LineCountingDOM { self.rcdom.mark_script_already_started(target); } - fn set_current_line(&self, line_number: u64) { - self.current_line.set(line_number); + fn set_current_source_position(&self, position: SourcePosition) { + self.current_line.set(position.line); } } diff --git a/rcdom/tests/source-positions-integration.rs b/rcdom/tests/source-positions-integration.rs new file mode 100644 index 00000000..17c7baee --- /dev/null +++ b/rcdom/tests/source-positions-integration.rs @@ -0,0 +1,224 @@ +//! Integration tests for the `source-positions` feature. +//! +//! Verifies that byte offsets flow correctly from `BufferQueue` through the +//! tokenizer and tree builder all the way into `TreeSink::set_current_source_position`, +//! and that the offsets correspond to the actual positions of element opening +//! tags in the source string. +//! +//! 2 Critical behaviours are under test: +//! +//! 1. When no explicit ,, tags are part of the payload +//! they get injected implicitly, they should not skew the byte offset. +//! 2. When the above tags are explicitly part of the payload, they should be part +//! of the count. + +#[cfg(feature = "source-positions")] +mod source_positions { + use html5ever::driver; + use html5ever::tendril::stream::TendrilSink; + use html5ever::tendril::StrTendril; + use html5ever::ExpandedName; + use html5ever::QualName; + use markup5ever::interface::{ElementFlags, NodeOrText, QuirksMode, SourcePosition, TreeSink}; + use markup5ever::Attribute; + use markup5ever_rcdom::{Handle, RcDom}; + use std::borrow::Cow; + use std::cell::{Cell, RefCell}; + + struct ByteCapturingDOM { + current_byte: Cell, + elements: RefCell>, + rcdom: RcDom, + } + + impl ByteCapturingDOM { + fn new() -> Self { + ByteCapturingDOM { + current_byte: Cell::new(0), + elements: RefCell::new(vec![]), + rcdom: RcDom::default(), + } + } + + fn content_elements(&self) -> Vec<(String, usize)> { + self.elements.borrow().clone() + } + } + + impl TreeSink for ByteCapturingDOM { + type Handle = Handle; + type Output = Self; + + type ElemName<'a> = ExpandedName<'a>; + + fn finish(self) -> Self { + self + } + + fn parse_error(&self, msg: Cow<'static, str>) { + self.rcdom.parse_error(msg); + } + + fn get_document(&self) -> Handle { + self.rcdom.get_document() + } + + fn elem_name<'a>(&'a self, target: &'a Handle) -> ExpandedName<'a> { + self.rcdom.elem_name(target) + } + + fn create_element( + &self, + name: QualName, + attrs: Vec, + flags: ElementFlags, + ) -> Handle { + self.elements + .borrow_mut() + .push((name.local.to_string(), self.current_byte.get())); + self.rcdom.create_element(name, attrs, flags) + } + + fn create_comment(&self, text: StrTendril) -> Handle { + self.rcdom.create_comment(text) + } + + fn create_pi(&self, target: StrTendril, content: StrTendril) -> Handle { + self.rcdom.create_pi(target, content) + } + + fn append(&self, parent: &Handle, child: NodeOrText) { + self.rcdom.append(parent, child) + } + + fn append_based_on_parent_node( + &self, + element: &Handle, + prev_element: &Handle, + child: NodeOrText, + ) { + self.rcdom + .append_based_on_parent_node(element, prev_element, child) + } + + fn append_doctype_to_document( + &self, + name: StrTendril, + public_id: StrTendril, + system_id: StrTendril, + ) { + self.rcdom + .append_doctype_to_document(name, public_id, system_id); + } + + fn get_template_contents(&self, target: &Handle) -> Handle { + self.rcdom.get_template_contents(target) + } + + fn same_node(&self, x: &Handle, y: &Handle) -> bool { + self.rcdom.same_node(x, y) + } + + fn set_quirks_mode(&self, mode: QuirksMode) { + self.rcdom.set_quirks_mode(mode) + } + + fn append_before_sibling(&self, sibling: &Handle, child: NodeOrText) { + self.rcdom.append_before_sibling(sibling, child) + } + + fn add_attrs_if_missing(&self, target: &Handle, attrs: Vec) { + self.rcdom.add_attrs_if_missing(target, attrs); + } + + fn remove_from_parent(&self, target: &Handle) { + self.rcdom.remove_from_parent(target); + } + + fn reparent_children(&self, node: &Handle, new_parent: &Handle) { + self.rcdom.reparent_children(node, new_parent); + } + + fn set_current_source_position(&self, position: SourcePosition) { + if let Some(byte) = position.byte { + self.current_byte.set(byte); + } + } + } + + fn parse(input: &str) -> ByteCapturingDOM { + let sink = ByteCapturingDOM::new(); + driver::parse_document(sink, Default::default()).one(StrTendril::from(input)) + } + + #[test] + fn element_byte_offsets_match_source_positions() { + let result = parse("

hello

world
"); + let elems = result.content_elements(); + + assert_eq!( + elems.len(), + 5, + "expected html, head, body, p and div, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 0)); + assert_eq!(elems[2], ("body".to_string(), 0)); + assert_eq!(elems[3], ("p".to_string(), 0)); + assert_eq!(elems[4], ("div".to_string(), 12)); + } + + #[test] + fn nested_element_byte_offset() { + let result = parse("
x
"); + let elems = result.content_elements(); + + assert_eq!( + elems.len(), + 5, + "expected html, head, body, div and span, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 0)); + assert_eq!(elems[2], ("body".to_string(), 0)); + assert_eq!(elems[3], ("div".to_string(), 0)); + assert_eq!(elems[4], ("span".to_string(), 5)); + } + + #[test] + fn explicit_html_head_body_offsets() { + let result = parse("

hi

"); + let elems = result.content_elements(); + + assert_eq!( + elems.len(), + 4, + "expected html, head, body, p, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 6)); + assert_eq!(elems[2], ("body".to_string(), 19)); + assert_eq!(elems[3], ("p".to_string(), 25)); + } + + #[test] + fn multibyte_content_does_not_shift_subsequent_offsets() { + let result = parse("

café

next"); + let elems = result.content_elements(); + + assert_eq!( + elems.len(), + 5, + "expected html, head, body, p and span, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 0)); + assert_eq!(elems[2], ("body".to_string(), 0)); + assert_eq!(elems[3], ("p".to_string(), 0)); + assert_eq!(elems[4], ("span".to_string(), 12)); + } +}