diff --git a/Cargo.lock b/Cargo.lock index be32ab6bf..226ada33f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,18 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -2082,6 +2094,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37dafa75fcb2ca73939cf7e7f4967d30be122b5e1d10f2b344e0addca8fa9446" +dependencies = [ + "icu_locale", + "icu_provider", +] + [[package]] name = "icu_provider_baked" version = "2.1.1" @@ -2097,6 +2119,21 @@ dependencies = [ "zerotrie", ] +[[package]] +name = "icu_provider_blob" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcd377d80121ab5d320c90f914cf7145a6881556d886c2aa2c6f89cdae5f7a30" +dependencies = [ + "icu_provider", + "log", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + [[package]] name = "icu_provider_export" version = "2.1.1" @@ -2107,6 +2144,7 @@ dependencies = [ "icu_locale", "icu_provider", "icu_provider_baked", + "icu_provider_blob", "log", "writeable", ] @@ -2155,9 +2193,9 @@ dependencies = [ [[package]] name = "icu_segmenter" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5e7e9b540df15e53ca27f69b50e36e01b652584b40b3335ed65d18303834" +checksum = "a807a7488f3f758629ae86d99d9d30dce24da2fb2945d74c80a4f4a62c71db73" dependencies = [ "core_maths", "databake", @@ -3303,6 +3341,8 @@ dependencies = [ "icu_normalizer", "icu_properties", "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", "icu_segmenter", "linebender_resource_handle", "oxipng", @@ -3377,6 +3417,7 @@ dependencies = [ "fontique", "oxipng", "parley", + "parley_data", "parley_dev", "parley_draw", "peniko", @@ -3537,6 +3578,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" dependencies = [ "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 02a566aef..8b7a4ffe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,9 +45,10 @@ icu_normalizer = { version = "~2.1.1", default-features = false } icu_properties = { version = "~2.1.2", default-features = false } icu_provider = { version = "2.1.1", default-features = false } icu_provider_adapters = { version = "2.1.1", default-features = false } +icu_provider_blob = { version = "2.1.1", default-features = false, features = ["alloc"] } icu_provider_export = { version = "2.1.1", default-features = false } icu_provider_source = { version = "2.1.1", default-features = false } -icu_segmenter = { version = "~2.1.1", default-features = false } +icu_segmenter = { version = "~2.1.2", default-features = false } linebender_resource_handle = { version = "0.1.1", default-features = false } parley = { version = "0.7.0", default-features = false, path = "parley" } parley_data = { path = "parley_data", default-features = false } diff --git a/parley/Cargo.toml b/parley/Cargo.toml index b10b4dbb6..1171d89b4 100644 --- a/parley/Cargo.toml +++ b/parley/Cargo.toml @@ -23,6 +23,14 @@ libm = ["fontique/libm", "peniko/libm", "skrifa/libm", "dep:core_maths"] # Enables support for system font backends system = ["std", "fontique/system"] accesskit = ["dep:accesskit"] +## Enables runtime loading of segmenter models for language-specific word/line breaking. +## This allows applications to load additional segmentation data (e.g., for Thai, Lao, +## Khmer, Burmese, Chinese, Japanese) at runtime rather than baking it into the binary. +runtime-segmenter-data = [ + "dep:icu_provider_adapters", + "dep:icu_provider_blob", + "icu_segmenter/serde", +] [dependencies] skrifa = { workspace = true } @@ -38,6 +46,8 @@ icu_collections = { workspace = true } icu_normalizer = { workspace = true } icu_properties = { workspace = true } icu_provider = { workspace = true } +icu_provider_blob = { workspace = true, optional = true } +icu_provider_adapters = { workspace = true, optional = true } icu_segmenter = { workspace = true, features = ["auto"] } # Used in ICU4X baked data sources zerovec = { workspace = true } diff --git a/parley/src/analysis/mod.rs b/parley/src/analysis/mod.rs index 84d7badc7..b0e9266bc 100644 --- a/parley/src/analysis/mod.rs +++ b/parley/src/analysis/mod.rs @@ -26,6 +26,73 @@ use icu_segmenter::{ }; use parley_data::CompositeProps; +#[cfg(feature = "runtime-segmenter-data")] +use icu_provider::buf::AsDeserializingBufferProvider; +#[cfg(feature = "runtime-segmenter-data")] +use icu_provider_adapters::fork::{ + ForkByMarkerProvider, MultiForkByErrorProvider, predicates::IdentifierNotFoundPredicate, +}; + +/// Segmenter model data that can be loaded at runtime. +/// +/// This type wraps binary blob data containing LSTM models or dictionaries for language-specific word/line +/// segmentation. The blobs are exported from `parley_data`, and can be included at compile time or saved as files from +/// `parley_data` at build time and later loaded as files at runtime. +/// +/// # Example +/// +/// ```ignore +/// use parley_data::SegmenterModelData; +/// +/// // Load from a file +/// let blob = std::fs::read("Thai_codepoints_exclusive_model4_heavy.postcard")?; +/// let model = SegmenterModelData::from_blob(blob.into_boxed_slice())?; +/// +/// // Or embed at compile time +/// let model = SegmenterModelData::from_static(parley_data::bundled_models::THAI_LSTM)?; +/// ``` +#[cfg(feature = "runtime-segmenter-data")] +#[derive(Debug)] +pub struct SegmenterModelData { + pub(crate) provider: icu_provider_blob::BlobDataProvider, +} + +#[cfg(feature = "runtime-segmenter-data")] +impl SegmenterModelData { + /// Creates a new `SegmenterModelData` from an owned blob. + /// + /// The blob should be a postcard-serialized ICU4X data blob from `parley_data`. + pub fn from_blob(blob: alloc::boxed::Box<[u8]>) -> Result { + let provider = icu_provider_blob::BlobDataProvider::try_new_from_blob(blob)?; + Ok(Self { provider }) + } + + /// Creates a new `SegmenterModelData` from a static byte slice. + /// + /// This is useful for embedding model data directly in your binary using `include_bytes!()`. + pub fn from_static(blob: &'static [u8]) -> Result { + let provider = icu_provider_blob::BlobDataProvider::try_new_from_static_blob(blob)?; + Ok(Self { provider }) + } +} + +/// The buffer provider for all data loaded at runtime. +#[cfg(feature = "runtime-segmenter-data")] +struct RuntimeBufferProvider { + provider: + MultiForkByErrorProvider, + segmenter_mode: SegmenterMode, +} + +#[allow(unused)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SegmenterMode { + /// Use LSTM for SE Asian scripts, dictionary for CJK (default). + Auto, + /// Use dictionary for all complex scripts. + Dictionary, +} + pub(crate) struct AnalysisDataSources { grapheme_segmenter: GraphemeClusterSegmenter, word_segmenter: WordSegmenter, @@ -36,6 +103,9 @@ pub(crate) struct AnalysisDataSources { brackets: CodePointMapData, composite: CompositeProps, + + #[cfg(feature = "runtime-segmenter-data")] + runtime_buffer_provider: Option, } #[derive(Default)] @@ -46,7 +116,13 @@ struct LineSegmenters { } impl LineSegmenters { - fn get(&mut self, word_break_strength: WordBreak) -> LineSegmenterBorrowed<'_> { + fn get( + &mut self, + word_break_strength: WordBreak, + #[cfg(feature = "runtime-segmenter-data")] runtime_buffer_provider: Option< + &RuntimeBufferProvider, + >, + ) -> LineSegmenterBorrowed<'_> { let segmenter = match word_break_strength { WordBreak::Normal => &mut self.normal, WordBreak::KeepAll => &mut self.keep_all, @@ -62,7 +138,27 @@ impl LineSegmenters { WordBreak::KeepAll => LineBreakWordOption::KeepAll, }; line_break_opts.word_option = Some(word_break_strength_icu); - LineSegmenter::try_new_auto_unstable(&PROVIDER, line_break_opts) + + #[cfg(feature = "runtime-segmenter-data")] + if let Some(&RuntimeBufferProvider { + ref provider, + segmenter_mode, + }) = runtime_buffer_provider + { + let combined = + ForkByMarkerProvider::new(provider.as_deserializing(), &PROVIDER); + return match segmenter_mode { + SegmenterMode::Auto => { + LineSegmenter::try_new_auto_unstable(&combined, line_break_opts) + } + SegmenterMode::Dictionary => { + LineSegmenter::try_new_dictionary_unstable(&combined, line_break_opts) + } + } + .expect("Failed to create LineSegmenter"); + } + + LineSegmenter::try_new_for_non_complex_scripts_unstable(&PROVIDER, line_break_opts) .expect("Failed to create LineSegmenter") }) .as_borrowed() @@ -73,7 +169,7 @@ impl AnalysisDataSources { pub(crate) fn new() -> Self { Self { grapheme_segmenter: GraphemeClusterSegmenter::try_new_unstable(&PROVIDER).unwrap(), - word_segmenter: WordSegmenter::try_new_lstm_unstable( + word_segmenter: WordSegmenter::try_new_for_non_complex_scripts_unstable( &PROVIDER, WordBreakOptions::default(), ) @@ -84,6 +180,73 @@ impl AnalysisDataSources { script_short_name: PropertyNamesShort::