Skip to content

uucore: share a fluent resource between threads - dd perf + 45% #11220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oech3 wants to merge 1 commit into
base: main
Choose a base branch
from
+47 −25
Open

uucore: share a fluent resource between threads - dd perf + 45% #11220

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 47 additions & 25 deletions src/uucore/src/lib/mods/locale.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,19 @@ include!(concat!(env!("OUT_DIR"), "/embedded_locales.rs"));

// A struct to handle localization with optional English fallback
struct Localizer {
primary_bundle: FluentBundle<FluentResource>,
fallback_bundle: Option<FluentBundle<FluentResource>>,
primary_bundle: FluentBundle<&'static FluentResource>,
fallback_bundle: Option<FluentBundle<&'static FluentResource>>,
}

impl Localizer {
fn new(primary_bundle: FluentBundle<FluentResource>) -> Self {
fn new(primary_bundle: FluentBundle<&'static FluentResource>) -> Self {
Self {
primary_bundle,
fallback_bundle: None,
}
}

fn with_fallback(mut self, fallback_bundle: FluentBundle<FluentResource>) -> Self {
fn with_fallback(mut self, fallback_bundle: FluentBundle<&'static FluentResource>) -> Self {
self.fallback_bundle = Some(fallback_bundle);
self
}
Expand Down Expand Up @@ -107,7 +107,10 @@ impl Localizer {
}
}

// Global localizer stored in thread-local OnceLock
// Cache localizer. FluentBundle is not Sync while FluentResource is Sync
static UUCORE_FLUENT: OnceLock<FluentResource> = OnceLock::new();
static CHECKSUM_FLUENT: OnceLock<FluentResource> = OnceLock::new();
static UTIL_FLUENT: OnceLock<FluentResource> = OnceLock::new();
thread_local! {
static LOCALIZER: OnceLock<Localizer> = const { OnceLock::new() };
}
Expand Down Expand Up @@ -136,8 +139,8 @@ fn create_bundle(
locale: &LanguageIdentifier,
locales_dir: &Path,
util_name: &str,
) -> Result<FluentBundle<FluentResource>, LocalizationError> {
let mut bundle = FluentBundle::new(vec![locale.clone()]);
) -> Result<FluentBundle<&'static FluentResource>, LocalizationError> {
let mut bundle: FluentBundle<&'static FluentResource> = FluentBundle::new(vec![locale.clone()]);

// Disable Unicode directional isolate characters
bundle.set_use_isolating(false);
Expand All @@ -148,7 +151,7 @@ fn create_bundle(
.and_then(|locale_path| fs::read_to_string(locale_path).ok())
.and_then(|ftl| FluentResource::try_new(ftl).ok())
{
bundle.add_resource_overriding(resource);
bundle.add_resource_overriding(Box::leak(Box::new(resource)));
Copy link
Contributor Author

@oech3 oech3 Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is not actually serious memory leak as coreutils has short lifespan.

}
};

Expand Down Expand Up @@ -193,10 +196,11 @@ fn init_localization(
.expect("Default locale should always be valid");

// Try to create a bundle that combines common and utility-specific strings
let english_bundle = create_bundle(&default_locale, locales_dir, util_name).or_else(|_| {
// Fallback to embedded utility-specific and common strings
create_english_bundle_from_embedded(&default_locale, util_name)
})?;
let english_bundle: FluentBundle<&'static FluentResource> =
create_bundle(&default_locale, locales_dir, util_name).or_else(|_| {
// Fallback to embedded utility-specific and common strings
create_english_bundle_from_embedded(&default_locale, util_name)
})?;

let loc = if locale == &default_locale {
// If requesting English, just use English as primary (no fallback needed)
Expand All @@ -220,8 +224,18 @@ fn init_localization(
}

/// Helper function to parse FluentResource from content string
fn parse_fluent_resource(content: &str) -> Result<FluentResource, LocalizationError> {
FluentResource::try_new(content.to_string()).map_err(
fn parse_fluent_resource(
content: &str,
cache: &'static OnceLock<FluentResource>,
) -> Result<&'static FluentResource, LocalizationError> {
// globa cache breaks unit tests
if cfg!(not(test)) {
if let Some(res) = cache.get() {
return Ok(res);
}
}

let resource = FluentResource::try_new(content.to_string()).map_err(
|(_partial_resource, errs): (FluentResource, Vec<ParserError>)| {
if let Some(first_err) = errs.into_iter().next() {
let snippet = first_err
Expand All @@ -238,42 +252,48 @@ fn parse_fluent_resource(content: &str) -> Result<FluentResource, LocalizationEr
LocalizationError::LocalesDirNotFound("Parse error without details".to_string())
}
},
)
)?;
// globa cache breaks unit tests
if cfg!(not(test)) {
Ok(cache.get_or_init(|| resource))
} else {
Ok(Box::leak(Box::new(resource)))
}
}

/// Create a bundle from embedded English locale files with common uucore strings
fn create_english_bundle_from_embedded(
locale: &LanguageIdentifier,
util_name: &str,
) -> Result<FluentBundle<FluentResource>, LocalizationError> {
) -> Result<FluentBundle<&'static FluentResource>, LocalizationError> {
// Only support English from embedded files
if *locale != "en-US" {
return Err(LocalizationError::LocalesDirNotFound(
"Embedded locales only support en-US".to_string(),
));
}

let mut bundle = FluentBundle::new(vec![locale.clone()]);
let mut bundle: FluentBundle<&'static FluentResource> = FluentBundle::new(vec![locale.clone()]);
bundle.set_use_isolating(false);

// First, try to load common uucore strings
if let Some(uucore_content) = get_embedded_locale("uucore/en-US.ftl") {
let uucore_resource = parse_fluent_resource(uucore_content)?;
let uucore_resource = parse_fluent_resource(uucore_content, &UUCORE_FLUENT)?;
bundle.add_resource_overriding(uucore_resource);
}

// Checksum algorithms need locale messages from checksum_common
if util_name.ends_with("sum") {
if let Some(uucore_content) = get_embedded_locale("checksum_common/en-US.ftl") {
let uucore_resource = parse_fluent_resource(uucore_content)?;
let uucore_resource = parse_fluent_resource(uucore_content, &CHECKSUM_FLUENT)?;
bundle.add_resource_overriding(uucore_resource);
}
}

// Then, try to load utility-specific strings
let locale_key = format!("{util_name}/en-US.ftl");
if let Some(ftl_content) = get_embedded_locale(&locale_key) {
let resource = parse_fluent_resource(ftl_content)?;
let resource = parse_fluent_resource(ftl_content, &UTIL_FLUENT)?;
bundle.add_resource_overriding(resource);
}

Expand Down Expand Up @@ -428,7 +448,8 @@ pub fn setup_localization(p: &str) -> Result<(), LocalizationError> {
// No locales directory found, use embedded English with common strings directly
let default_locale = LanguageIdentifier::from_str(DEFAULT_LOCALE)
.expect("Default locale should always be valid");
let english_bundle = create_english_bundle_from_embedded(&default_locale, p)?;
let english_bundle: FluentBundle<&'static FluentResource> =
create_english_bundle_from_embedded(&default_locale, p)?;
let localizer = Localizer::new(english_bundle);

LOCALIZER.with(|lock| {
Expand Down Expand Up @@ -593,14 +614,15 @@ mod tests {
fn create_test_bundle(
locale: &LanguageIdentifier,
test_locales_dir: &Path,
) -> Result<FluentBundle<FluentResource>, LocalizationError> {
let mut bundle = FluentBundle::new(vec![locale.clone()]);
) -> Result<FluentBundle<&'static FluentResource>, LocalizationError> {
let mut bundle: FluentBundle<&'static FluentResource> =
FluentBundle::new(vec![locale.clone()]);
bundle.set_use_isolating(false);

// Only load from the test directory - no common strings or utility-specific paths
let locale_path = test_locales_dir.join(format!("{locale}.ftl"));
if let Ok(ftl_content) = fs::read_to_string(&locale_path) {
let resource = parse_fluent_resource(&ftl_content)?;
let resource = parse_fluent_resource(&ftl_content, &UUCORE_FLUENT)?;
bundle.add_resource_overriding(resource);
return Ok(bundle);
}
Expand Down Expand Up @@ -763,7 +785,7 @@ invalid-syntax = This is { $missing
#[test]
fn test_localizer_format_primary_bundle() {
let temp_dir = create_test_locales_dir();
let en_bundle = create_test_bundle(
let en_bundle: FluentBundle<&'static FluentResource> = create_test_bundle(
&LanguageIdentifier::from_str("en-US").unwrap(),
temp_dir.path(),
)
Expand Down
Loading