Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
a94ab0d
test_export_language_identifier_display_names
sffc Aug 7, 2026
df937bf
🤖 Optimize CoverageByXPathLevels data model and trie queries for disp…
sffc Aug 7, 2026
70c1aa4
make the test slightly faster
sffc Aug 7, 2026
281345f
🤖 Fix clippy warnings in displaynames coverage module
sffc Aug 7, 2026
4eba8a7
reduce diff
sffc Aug 7, 2026
f992f45
🤖 Fix redundant closure clippy warning in e2e test
sffc Aug 7, 2026
26b0119
🤖 Move e2e benchmark test to standalone benches/displaynames.rs binary
sffc Aug 10, 2026
a66c6a3
🤖 Simplify parse_cldr_xpath string splitting using split_once
sffc Aug 10, 2026
2e62ba2
🤖 Refactor CoverageByXPathLevels field selection to use CoverageCateg…
sffc Aug 10, 2026
923036a
🤖 Implement custom Serde deserializer for CoverageByXPathLevels to el…
sffc Aug 10, 2026
cf7f0c5
🤖 Remove is_language parameter and use writeable::adapters::Replace u…
sffc Aug 10, 2026
a15a6c9
🤖 Update macro and function doc comments and restore CoverageByXPathL…
sffc Aug 10, 2026
8ae0339
🤖 Handle Unknown variants in Alt::as_str and Menu::as_str by returnin…
sffc Aug 10, 2026
3b5fd98
🤖 Normalize subtags using icu_locale_core during deserialization and …
sffc Aug 10, 2026
3146bfe
🤖 Add module documentation to displaynames benchmark
sffc Aug 10, 2026
cd13bf9
🤖 Clarify CoverageCategoryLevels doc comment
sffc Aug 10, 2026
a30753e
🤖 Use named struct fields in XPathArraySeed and XPathArrayVisitor
sffc Aug 10, 2026
efe4501
🤖 Import Cow at top of coverage_experimental module
sffc Aug 10, 2026
80b422a
🤖 Add docstrings to Alt::as_str and Menu::as_str
sffc Aug 10, 2026
a2edf65
🤖 Add comment explaining non-ASCII XPath exclusion in visit_seq
sffc Aug 10, 2026
2e747fe
🤖 Hoist root_levels lookup outside per-locale loop in displaynames it…
sffc Aug 10, 2026
6e649fc
🤖 Import subtag types at file top in coverage_experimental
sffc Aug 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion provider/source/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ num-traits = { workspace = true, optional = true }
[dev-dependencies]
postcard = { workspace = true, features = ["alloc"] }
icu_provider_adapters = { workspace = true }
icu_provider_export = { workspace = true, features = ["fs_exporter", "baked_exporter", "rayon"] }
icu_provider_export = { workspace = true, features = ["fs_exporter", "baked_exporter", "rayon", "blob_exporter"] }
icu_provider = { workspace = true, features = ["deserialize_postcard_1"] }
icu_segmenter = { path = "../../components/segmenter", features = ["lstm"] }
simple_logger = { workspace = true }
Expand Down Expand Up @@ -105,5 +105,15 @@ icu_experimental = []
skip_feature_sets = [["use_icu4c"], ["use_wasm"]]
max_combination_size = 3

[lib]
bench = false # This option is required for Benchmark CI

# Standalone benchmark binary for display names data export performance.
# We set `harness = false` so that it runs as a standalone binary with main() when
# executing `cargo bench`, but is NOT executed during `cargo test` / `cargo make ci-job-test`.
[[bench]]
name = "displaynames"
harness = false

[lints]
workspace = true
49 changes: 49 additions & 0 deletions provider/source/benches/displaynames.rs

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see the value of this test. It was probably useful while you were working on performance improvements, but I don't think it should be checked in.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved it to a benchmark

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also don't think it's useful as a benchmark

in any case, it should use our standard benchmark scaffolding

@sffc sffc Aug 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was intentional in making it a plain main fn. It is too slow for criterion, and I don't want to pull in other dependencies.

I think this is valuable as a benchmark because this covers a lot of datagen code paths, and improving the performance of this benchmark can improve the performance of datagen in general. Datagen in general is getting quite slow, so benchmarks seem warranted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// This file is part of ICU4X. For terms of use, please see the file
Comment thread
sffc marked this conversation as resolved.
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! Standalone benchmark measuring performance of exporting display names data
//! across modern coverage locales.
//!
//! Run using:
//! ```text
//! cargo bench -p icu_provider_source --bench displaynames
//! ```

use icu_provider_export::blob_exporter::BlobExporter;
use icu_provider_export::prelude::*;
use icu_provider_source::SourceDataProvider;

fn main() {
let t0 = std::time::Instant::now();
let provider = SourceDataProvider::new();
let mut blob_bytes = Vec::new();
let exporter = BlobExporter::new_with_sink(Box::new(&mut blob_bytes));

let modern_locales = provider
.locales_for_coverage_levels([icu_provider_source::CoverageLevel::Modern])
.unwrap();

ExportDriver::new(
modern_locales
.into_iter()
.map(DataLocaleFamily::without_descendants),
DeduplicationStrategy::None.into(),
LocaleFallbacker::try_new_unstable(&provider).unwrap(),
)
.with_markers(
icu::experimental::provider::MARKERS
.iter()
.copied()
.filter(|info| info.id.name().starts_with("LocaleNames")),
)
.export(&provider, exporter)
.unwrap();

let elapsed = t0.elapsed();
println!(
"displaynames bench: {:.3?} s, blob size: {} bytes",
elapsed.as_secs_f64(),
blob_bytes.len()
);
}
29 changes: 29 additions & 0 deletions provider/source/src/cldr_serde/displaynames/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,24 @@ pub(crate) enum Alt {
Menu,
}

impl Alt {
/// Returns the string representation of the `Alt` variant, or `None` if `Unknown`.
pub fn as_str(self) -> Option<&'static str> {
match self {
Alt::Short => Some("short"),
Alt::Long => Some("long"),
Alt::Variant => Some("variant"),
Alt::StandAlone => Some("stand-alone"),
Alt::Official => Some("official"),
Alt::Secondary => Some("secondary"),
Alt::Biot => Some("biot"),
Alt::Chagos => Some("chagos"),
Alt::Menu => Some("menu"),
Alt::Unknown => None,
}
}
}

impl FromStr for Alt {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
Expand All @@ -57,6 +75,17 @@ pub(crate) enum Menu {
Extension,
}

impl Menu {
/// Returns the string representation of the `Menu` variant, or `None` if `Unknown`.
pub fn as_str(self) -> Option<&'static str> {
match self {
Menu::Core => Some("core"),
Menu::Extension => Some("extension"),
Menu::Unknown => None,
}
}
}

impl FromStr for Menu {
type Err = ();
Comment thread
sffc marked this conversation as resolved.
Comment thread
sffc marked this conversation as resolved.
fn from_str(s: &str) -> Result<Self, Self::Err> {
Expand Down
Loading
Loading