From 1a67974a23bcd112fd715cee2f065752d674941c Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 1 Feb 2026 17:37:41 -0500 Subject: [PATCH 01/15] Add locale data API --- Cargo.lock | 43 ++++ Cargo.toml | 1 + parley/Cargo.toml | 10 + parley/src/analysis/mod.rs | 191 ++++++++++++++++-- parley/src/analysis/provider.rs | 2 + parley/src/context.rs | 67 ++++++ parley/src/lib.rs | 2 + parley_data/Cargo.toml | 1 + parley_data/src/generated/icu4x_data/mod.rs | 4 + .../segmenter_dictionary_auto_v1.rs.data | 56 +++++ .../segmenter_dictionary_extended_v1.rs.data | 56 +++++ ...codepoints_exclusive_model4_heavy.postcard | Bin 0 -> 127602 bytes ...codepoints_exclusive_model4_heavy.postcard | Bin 0 -> 110898 bytes ...codepoints_exclusive_model4_heavy.postcard | Bin 0 -> 108391 bytes ...codepoints_exclusive_model4_heavy.postcard | Bin 0 -> 108559 bytes .../segmenter_models/burmesedict.postcard | Bin 0 -> 493769 bytes .../segmenter_models/cjdict.postcard | Bin 0 -> 2043610 bytes .../segmenter_models/khmerdict.postcard | Bin 0 -> 834877 bytes .../segmenter_models/laodict.postcard | Bin 0 -> 328961 bytes .../segmenter_models/thaidict.postcard | Bin 0 -> 261480 bytes parley_data/src/lib.rs | 97 +++++++++ parley_data_gen/Cargo.toml | 2 +- parley_data_gen/README.md | 2 +- parley_data_gen/src/lib.rs | 47 +++++ parley_dev/README.md | 1 + .../fonts/noto_fonts/NotoSansThai-Regular.ttf | Bin 0 -> 42036 bytes parley_tests/Cargo.toml | 3 +- ...ript_line_break_with_multiple_models-0.png | Bin 0 -> 5224 bytes ...xed_script_line_break_with_no_models-0.png | Bin 0 -> 5355 bytes .../thai_line_break_with_model-0.png | Bin 0 -> 3525 bytes .../thai_line_break_without_model-0.png | Bin 0 -> 4796 bytes parley_tests/tests/mod.rs | 1 + parley_tests/tests/segmenter.rs | 102 ++++++++++ parley_tests/tests/util/env.rs | 5 + 34 files changed, 677 insertions(+), 16 deletions(-) create mode 100644 parley_data/src/generated/icu4x_data/segmenter_dictionary_auto_v1.rs.data create mode 100644 parley_data/src/generated/icu4x_data/segmenter_dictionary_extended_v1.rs.data create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/Burmese_codepoints_exclusive_model4_heavy.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/Khmer_codepoints_exclusive_model4_heavy.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/Lao_codepoints_exclusive_model4_heavy.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/Thai_codepoints_exclusive_model4_heavy.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/burmesedict.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/cjdict.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/khmerdict.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/laodict.postcard create mode 100644 parley_data/src/generated/icu4x_data/segmenter_models/thaidict.postcard create mode 100644 parley_dev/assets/fonts/noto_fonts/NotoSansThai-Regular.ttf create mode 100644 parley_tests/snapshots/mixed_script_line_break_with_multiple_models-0.png create mode 100644 parley_tests/snapshots/mixed_script_line_break_with_no_models-0.png create mode 100644 parley_tests/snapshots/thai_line_break_with_model-0.png create mode 100644 parley_tests/snapshots/thai_line_break_without_model-0.png create mode 100644 parley_tests/tests/segmenter.rs diff --git a/Cargo.lock b/Cargo.lock index be32ab6bf..1fb5352af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,18 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -2082,6 +2094,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37dafa75fcb2ca73939cf7e7f4967d30be122b5e1d10f2b344e0addca8fa9446" +dependencies = [ + "icu_locale", + "icu_provider", +] + [[package]] name = "icu_provider_baked" version = "2.1.1" @@ -2097,6 +2119,21 @@ dependencies = [ "zerotrie", ] +[[package]] +name = "icu_provider_blob" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcd377d80121ab5d320c90f914cf7145a6881556d886c2aa2c6f89cdae5f7a30" +dependencies = [ + "icu_provider", + "log", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + [[package]] name = "icu_provider_export" version = "2.1.1" @@ -2107,6 +2144,7 @@ dependencies = [ "icu_locale", "icu_provider", "icu_provider_baked", + "icu_provider_blob", "log", "writeable", ] @@ -3303,6 +3341,8 @@ dependencies = [ "icu_normalizer", "icu_properties", "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", "icu_segmenter", "linebender_resource_handle", "oxipng", @@ -3377,6 +3417,7 @@ dependencies = [ "fontique", "oxipng", "parley", + "parley_data", "parley_dev", "parley_draw", "peniko", @@ -3537,6 +3578,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" dependencies = [ "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 02a566aef..f445dfa40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ icu_normalizer = { version = "~2.1.1", default-features = false } icu_properties = { version = "~2.1.2", default-features = false } icu_provider = { version = "2.1.1", default-features = false } icu_provider_adapters = { version = "2.1.1", default-features = false } +icu_provider_blob = { version = "2.1.1", default-features = false, features = ["alloc"] } icu_provider_export = { version = "2.1.1", default-features = false } icu_provider_source = { version = "2.1.1", default-features = false } icu_segmenter = { version = "~2.1.1", default-features = false } diff --git a/parley/Cargo.toml b/parley/Cargo.toml index b10b4dbb6..1171d89b4 100644 --- a/parley/Cargo.toml +++ b/parley/Cargo.toml @@ -23,6 +23,14 @@ libm = ["fontique/libm", "peniko/libm", "skrifa/libm", "dep:core_maths"] # Enables support for system font backends system = ["std", "fontique/system"] accesskit = ["dep:accesskit"] +## Enables runtime loading of segmenter models for language-specific word/line breaking. +## This allows applications to load additional segmentation data (e.g., for Thai, Lao, +## Khmer, Burmese, Chinese, Japanese) at runtime rather than baking it into the binary. +runtime-segmenter-data = [ + "dep:icu_provider_adapters", + "dep:icu_provider_blob", + "icu_segmenter/serde", +] [dependencies] skrifa = { workspace = true } @@ -38,6 +46,8 @@ icu_collections = { workspace = true } icu_normalizer = { workspace = true } icu_properties = { workspace = true } icu_provider = { workspace = true } +icu_provider_blob = { workspace = true, optional = true } +icu_provider_adapters = { workspace = true, optional = true } icu_segmenter = { workspace = true, features = ["auto"] } # Used in ICU4X baked data sources zerovec = { workspace = true } diff --git a/parley/src/analysis/mod.rs b/parley/src/analysis/mod.rs index 84d7badc7..38d643ad0 100644 --- a/parley/src/analysis/mod.rs +++ b/parley/src/analysis/mod.rs @@ -26,6 +26,73 @@ use icu_segmenter::{ }; use parley_data::CompositeProps; +#[cfg(feature = "runtime-segmenter-data")] +use icu_provider::buf::AsDeserializingBufferProvider; +#[cfg(feature = "runtime-segmenter-data")] +use icu_provider_adapters::fork::{ + ForkByMarkerProvider, MultiForkByErrorProvider, predicates::IdentifierNotFoundPredicate, +}; + +/// Segmenter model data that can be loaded at runtime. +/// +/// This type wraps binary blob data containing LSTM or dictionary models for language-specific word/line segmentation. +/// The blobs are exported from `parley_data` with the `bundled-segmenter-models` feature, and can be included at +/// compile time or exported from `parley_data` and loaded as files at runtime. +/// +/// # Example +/// +/// ```ignore +/// use parley_data::SegmenterModelData; +/// +/// // Load from a file +/// let blob = std::fs::read("Thai_codepoints_exclusive_model4_heavy.postcard")?; +/// let model = SegmenterModelData::from_blob(blob.into_boxed_slice())?; +/// +/// // Or embed at compile time +/// let model = SegmenterModelData::from_static(parley_data::bundled_models::THAI_LSTM)?; +/// ``` +#[cfg(feature = "runtime-segmenter-data")] +#[derive(Debug)] +pub struct SegmenterModelData { + pub(crate) provider: icu_provider_blob::BlobDataProvider, +} + +#[cfg(feature = "runtime-segmenter-data")] +impl SegmenterModelData { + /// Creates a new `SegmenterModelData` from an owned blob. + /// + /// The blob should be a postcard-serialized ICU4X data blob from `parley_data`. + pub fn from_blob(blob: alloc::boxed::Box<[u8]>) -> Result { + let provider = icu_provider_blob::BlobDataProvider::try_new_from_blob(blob)?; + Ok(Self { provider }) + } + + /// Creates a new `SegmenterModelData` from a static byte slice. + /// + /// This is useful for embedding model data directly in your binary using `include_bytes!()`. + pub fn from_static(blob: &'static [u8]) -> Result { + let provider = icu_provider_blob::BlobDataProvider::try_new_from_static_blob(blob)?; + Ok(Self { provider }) + } +} + +/// The buffer provider for all data loaded at runtime. +#[cfg(feature = "runtime-segmenter-data")] +struct RuntimeBufferProvider { + provider: + MultiForkByErrorProvider, + segmenter_mode: SegmenterMode, +} + +#[allow(unused)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SegmenterMode { + /// Use LSTM for SE Asian scripts, dictionary for CJK (default). + Auto, + /// Use dictionary for all complex scripts. + Dictionary, +} + pub(crate) struct AnalysisDataSources { grapheme_segmenter: GraphemeClusterSegmenter, word_segmenter: WordSegmenter, @@ -36,6 +103,9 @@ pub(crate) struct AnalysisDataSources { brackets: CodePointMapData, composite: CompositeProps, + + #[cfg(feature = "runtime-segmenter-data")] + runtime_buffer_provider: Option, } #[derive(Default)] @@ -46,7 +116,12 @@ struct LineSegmenters { } impl LineSegmenters { - fn get(&mut self, word_break_strength: WordBreak) -> LineSegmenterBorrowed<'_> { + #[cfg(feature = "runtime-segmenter-data")] + fn get( + &mut self, + runtime_buffer_provider: Option<&RuntimeBufferProvider>, + word_break_strength: WordBreak, + ) -> LineSegmenterBorrowed<'_> { let segmenter = match word_break_strength { WordBreak::Normal => &mut self.normal, WordBreak::KeepAll => &mut self.keep_all, @@ -55,25 +130,64 @@ impl LineSegmenters { segmenter .get_or_insert_with(|| { - let mut line_break_opts = LineBreakOptions::default(); - let word_break_strength_icu = match word_break_strength { - WordBreak::Normal => LineBreakWordOption::Normal, - WordBreak::BreakAll => LineBreakWordOption::BreakAll, - WordBreak::KeepAll => LineBreakWordOption::KeepAll, - }; - line_break_opts.word_option = Some(word_break_strength_icu); + let line_break_opts = Self::make_line_break_opts(word_break_strength); + if let Some(&RuntimeBufferProvider { + ref provider, + segmenter_mode, + }) = runtime_buffer_provider + { + let combined = + ForkByMarkerProvider::new(provider.as_deserializing(), &PROVIDER); + match segmenter_mode { + SegmenterMode::Auto => { + LineSegmenter::try_new_auto_unstable(&combined, line_break_opts) + } + SegmenterMode::Dictionary => { + LineSegmenter::try_new_dictionary_unstable(&combined, line_break_opts) + } + } + } else { + LineSegmenter::try_new_auto_unstable(&PROVIDER, line_break_opts) + } + .expect("Failed to create LineSegmenter") + }) + .as_borrowed() + } + + #[cfg(not(feature = "runtime-segmenter-data"))] + fn get(&self, word_break_strength: WordBreak) -> LineSegmenterBorrowed<'_> { + let segmenter = match word_break_strength { + WordBreak::Normal => &self.normal, + WordBreak::KeepAll => &self.keep_all, + WordBreak::BreakAll => &self.break_all, + }; + + segmenter + .get_or_init(|| { + let line_break_opts = Self::make_line_break_opts(word_break_strength); LineSegmenter::try_new_auto_unstable(&PROVIDER, line_break_opts) .expect("Failed to create LineSegmenter") }) .as_borrowed() } + + fn make_line_break_opts(word_break_strength: WordBreak) -> LineBreakOptions<'static> { + let mut line_break_opts = LineBreakOptions::default(); + let word_break_strength_icu = match word_break_strength { + WordBreak::Normal => LineBreakWordOption::Normal, + WordBreak::BreakAll => LineBreakWordOption::BreakAll, + WordBreak::KeepAll => LineBreakWordOption::KeepAll, + }; + line_break_opts.word_option = Some(word_break_strength_icu); + line_break_opts + } } impl AnalysisDataSources { pub(crate) fn new() -> Self { Self { grapheme_segmenter: GraphemeClusterSegmenter::try_new_unstable(&PROVIDER).unwrap(), - word_segmenter: WordSegmenter::try_new_lstm_unstable( + word_segmenter: WordSegmenter::try_new_auto_unstable( &PROVIDER, WordBreakOptions::default(), ) @@ -84,7 +198,44 @@ impl AnalysisDataSources { script_short_name: PropertyNamesShort::