Move from unic_langid to icu_locid

Author: zbraniecki

Cargo.toml

@@ -7,11 +7,10 @@ version = "0.13.0"
 authors = ["Zibi Braniecki <gandalf@mozilla.com>"]
 homepage = "http://projectfluent.org/"
 license = "Apache-2.0"
-license-file = "LICENSE"
 repository = "https://github.com/projectfluent/fluent-langneg-rs"
 readme = "README.md"
 categories = ["internationalization", "localization"]
-edition = "2018"
+edition = "2021"
 include = [
 	"src/**/*",
 	"benches/*.rs",
@@ -26,13 +25,11 @@ coveralls = { repository = "projectfluent/fluent-langneg-rs", branch = "master",
 maintenance = { status = "actively-developed" }
 
 [dependencies]
-unic-langid = "0.9"
+icu_locid = "1.4"
 
 [dev-dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-unic-langid = { version = "0.9", features = ["macros"] }
-unic-locale = { version = "0.9", features = ["macros"] }
 criterion = "0.3"
 
 [[bench]]
@@ -41,4 +38,3 @@ harness = false
 
 [features]
 default = []
-cldr = ["unic-langid/likelysubtags"]

benches/negotiate.rs

@@ -5,7 +5,7 @@ use criterion::Criterion;
 use fluent_langneg::convert_vec_str_to_langids_lossy;
 use fluent_langneg::negotiate_languages;
 
-use unic_langid::LanguageIdentifier;
+use icu_locid::LanguageIdentifier;
 
 #[no_mangle]
 #[inline(never)]

src/accepted_languages.rs

@@ -8,7 +8,7 @@
 //! use fluent_langneg::NegotiationStrategy;
 //! use fluent_langneg::parse_accepted_languages;
 //! use fluent_langneg::convert_vec_str_to_langids_lossy;
-//! use unic_langid::LanguageIdentifier;
+//! use icu_locid::LanguageIdentifier;
 //!
 //! let requested = parse_accepted_languages("de-AT;0.9,de-DE;0.8,de;0.7;en-US;0.5");
 //! let available = convert_vec_str_to_langids_lossy(&["fr", "pl", "de", "en-US"]);
@@ -30,11 +30,11 @@
 //! language negotiation only uses the order of locales, not the weights.
 //!
 
-use unic_langid::LanguageIdentifier;
+use icu_locid::LanguageIdentifier;
 
 pub fn parse(s: &str) -> Vec<LanguageIdentifier> {
     s.split(',')
-        .map(|t| t.trim().split(';').nth(0).unwrap())
+        .map(|t| t.trim().split(';').next().unwrap())
         .filter(|t| !t.is_empty())
         .filter_map(|t| t.parse().ok())
         .collect()

src/lib.rs

@@ -22,18 +22,18 @@ pub use accepted_languages::parse as parse_accepted_languages;
 pub use negotiate::negotiate_languages;
 pub use negotiate::NegotiationStrategy;
 
-use unic_langid::{LanguageIdentifier, LanguageIdentifierError};
+use icu_locid::{LanguageIdentifier, ParserError};
 
 pub fn convert_vec_str_to_langids<'a, I, J>(
     input: I,
-) -> Result<Vec<LanguageIdentifier>, LanguageIdentifierError>
+) -> Result<Vec<LanguageIdentifier>, ParserError>
 where
     I: IntoIterator<Item = J>,
     J: AsRef<[u8]> + 'a,
 {
     input
         .into_iter()
-        .map(|s| LanguageIdentifier::from_bytes(s.as_ref()))
+        .map(|s| LanguageIdentifier::try_from_bytes(s.as_ref()))
         .collect()
 }
 
@@ -44,6 +44,6 @@ where
 {
     input
         .into_iter()
-        .filter_map(|t| LanguageIdentifier::from_bytes(t.as_ref()).ok())
+        .filter_map(|t| LanguageIdentifier::try_from_bytes(t.as_ref()).ok())
         .collect()
 }

src/negotiate/likely_subtags.rs

@@ -1,4 +1,4 @@
-use unic_langid::LanguageIdentifier;
+use icu_locid::LanguageIdentifier;
 
 static REGION_MATCHING_KEYS: &[&str] = &[
     "az", "bg", "cs", "de", "es", "fi", "fr", "hu", "it", "lt", "lv", "nl", "pl", "ro", "ru",
@@ -22,7 +22,7 @@ impl MockLikelySubtags for LanguageIdentifier {
                 let lang = self.language;
 
                 for subtag in REGION_MATCHING_KEYS {
-                    if lang == *subtag {
+                    if lang.strict_cmp(subtag.as_bytes()).is_eq() {
                         self.region = Some(subtag.parse().unwrap());
                         return true;
                     }

src/negotiate/mod.rs

@@ -14,7 +14,7 @@
 //! use fluent_langneg::negotiate_languages;
 //! use fluent_langneg::NegotiationStrategy;
 //! use fluent_langneg::convert_vec_str_to_langids_lossy;
-//! use unic_langid::LanguageIdentifier;
+//! use icu_locid::LanguageIdentifier;
 //!
 //! let requested = convert_vec_str_to_langids_lossy(&["pl", "fr", "en-US"]);
 //! let available = convert_vec_str_to_langids_lossy(&["it", "de", "fr", "en-GB", "en_US"]);
@@ -119,7 +119,7 @@
 //! ```
 //!
 
-use unic_langid::LanguageIdentifier;
+use icu_locid::LanguageIdentifier;
 
 #[cfg(not(feature = "cldr"))]
 mod likely_subtags;
@@ -133,6 +133,40 @@ pub enum NegotiationStrategy {
     Lookup,
 }
 
+fn subtag_matches<P: PartialEq>(
+    subtag1: &Option<P>,
+    subtag2: &Option<P>,
+    as_range1: bool,
+    as_range2: bool,
+) -> bool {
+    (as_range1 && subtag1.is_none()) || (as_range2 && subtag2.is_none()) || subtag1 == subtag2
+}
+
+fn matches(
+    lid1: &LanguageIdentifier,
+    lid2: &LanguageIdentifier,
+    range1: bool,
+    range2: bool,
+) -> bool {
+    let language = (range1 && lid1.language.is_empty())
+        || (range2 && lid2.language.is_empty())
+        || lid1.language == lid2.language;
+    if !language {
+        return false;
+    }
+    let script = subtag_matches(&lid1.script, &lid2.script, range1, range2);
+    if !script {
+        return false;
+    }
+    let region = subtag_matches(&lid1.region, &lid2.region, range1, range2);
+    if !region {
+        return false;
+    }
+    (range1 && lid1.variants.is_empty())
+        || (range2 && lid2.variants.is_empty())
+        || lid1.variants == lid2.variants
+}
+
 pub fn filter_matches<'a, R: 'a + AsRef<LanguageIdentifier>, A: 'a + AsRef<LanguageIdentifier>>(
     requested: &[R],
     available: &'a [A],
@@ -152,10 +186,7 @@ pub fn filter_matches<'a, R: 'a + AsRef<LanguageIdentifier>, A: 'a + AsRef<Langu
                         return true;
                     }
 
-                    if locale
-                        .as_ref()
-                        .matches(&req, $self_as_range, $other_as_range)
-                    {
+                    if matches(locale.as_ref(), &req, $self_as_range, $other_as_range) {
                         match_found = true;
                         supported_locales.push(*locale);
                         return false;
@@ -191,7 +222,7 @@ pub fn filter_matches<'a, R: 'a + AsRef<LanguageIdentifier>, A: 'a + AsRef<Langu
         }
 
         // 4) Try to match against a variant as a range
-        req.clear_variants();
+        req.variants.clear();
         test_strategy!(true, true);
 
         // 5) Try to match against the likely subtag without region

tests/lib.rs

@@ -7,9 +7,7 @@ use fluent_langneg::convert_vec_str_to_langids_lossy;
 use fluent_langneg::negotiate_languages;
 use fluent_langneg::parse_accepted_languages;
 use fluent_langneg::NegotiationStrategy;
-use unic_langid::langid;
-use unic_langid::LanguageIdentifier;
-use unic_locale::locale;
+use icu_locid::{langid, locale, LanguageIdentifier, Locale};
 
 use serde::{Deserialize, Serialize};
 
@@ -190,7 +188,7 @@ fn locale_matching() {
     let loc_de_at = locale!("de-AT-u-hc-h24");
     let loc_en = locale!("en-u-ca-buddhist");
     let loc_de = locale!("de");
-    let loc_pl = locale!("pl-x-private");
+    let loc_pl: Locale = "pl-x-private".parse().unwrap();
 
     assert_eq!(
         negotiate_languages(