Skip to content

Commit eddbc65

Browse files
filipeforattinifilipeforattini
committed
feat(rust): normalize Unicode strings
1 parent e706d61 commit eddbc65

10 files changed

Lines changed: 71 additions & 5 deletions

File tree

‎packages/compiler/src/backend/rust/lib-calls.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,9 @@ export function emitRustLibCall(expr: RustLibCallExpr, context: RustLibCallConte
311311
if (expr.fn === "str.substr" && expr.args.length === 3 && arg?.type.kind === "string" && secondArg !== undefined && expr.args[2] !== undefined) {
312312
return `runtime::string_substr(&(${context.emitExpr(arg)}), ${context.emitExpr(secondArg)}, ${context.emitExpr(expr.args[2])})`;
313313
}
314+
if (expr.fn === "str.normalize" && expr.args.length === 2 && arg?.type.kind === "string" && secondArg?.type.kind === "string") {
315+
return `runtime::string_normalize(&(${context.emitExpr(arg)}), &(${context.emitExpr(secondArg)}))`;
316+
}
314317
if (expr.fn === "str.concatDyn" && expr.args.length === 2 && arg?.type.kind === "string" && secondArg?.type.kind === "dyn") {
315318
const receiver = context.nextTemporary();
316319
const values = context.nextTemporary();

‎packages/compiler/src/coverage/backend-libcalls.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,6 +1138,7 @@ export const BACKEND_LIB_CALLS: Readonly<Record<string, readonly BackendId[]>> =
11381138
"str.decodeUriComponent": ["c", "llvm", "rust"],
11391139
"str.encodeUri": ["c", "llvm", "rust"],
11401140
"str.encodeUriComponent": ["c", "llvm", "rust"],
1141+
"str.normalize": ["rust"],
11411142
"str.substr": ["rust"],
11421143
"strdec.end": ["c", "llvm", "rust"],
11431144
"strdec.next": ["c", "llvm", "rust"],

‎packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,8 @@ export function lowerStringMethodCall(lowerer: Lowerer, call: ts.CallExpression,
550550
const substr = access.name.text === "substr" && lowerer.nativeStringSubstr;
551551
const concatDyn = access.name.text === "concat" && lowerer.nativeStringConcatDyn &&
552552
argumentNodes.length === 1 && ts.isSpreadElement(argumentNodes[0]!);
553-
if (!entry && !indexMethod && !substr && !concatDyn) return null;
553+
const normalize = access.name.text === "normalize" && lowerer.nativeStringNormalize;
554+
if (!entry && !indexMethod && !substr && !concatDyn && !normalize) return null;
554555
// A validated dyn receiver (`pkg.name.replace(...)` on a JSON.parse
555556
// value) arrives pre-extracted through `dynReceiver`; its checker type
556557
// is `any`, so the type/symbol gates don't apply — the dyn value's
@@ -570,8 +571,8 @@ export function lowerStringMethodCall(lowerer: Lowerer, call: ts.CallExpression,
570571
}
571572
// The lib declares optional parameters beyond some lowered forms; fence
572573
// those arities instead of passing arguments the runtime doesn't take.
573-
const minArgs = concatDyn ? 1 : substr ? 0 : (entry?.minArgs ?? 0);
574-
const maxArgs = concatDyn ? 1 : substr ? 2 : (entry?.maxArgs ?? 1);
574+
const minArgs = concatDyn ? 1 : substr || normalize ? 0 : (entry?.minArgs ?? 0);
575+
const maxArgs = concatDyn ? 1 : substr ? 2 : normalize ? 1 : (entry?.maxArgs ?? 1);
575576
if (argumentNodes.length < minArgs || argumentNodes.length > maxArgs) {
576577
lowerer.noLowering(
577578
`.${access.name.text} with ${argumentNodes.length} argument${argumentNodes.length === 1 ? "" : "s"} on strings`,
@@ -584,6 +585,13 @@ export function lowerStringMethodCall(lowerer: Lowerer, call: ts.CallExpression,
584585
const loc = locOf(call);
585586
if (indexMethod) return lowerStringIndexCall(lowerer, call, indexMethod, receiver, access.expression, argumentNodes);
586587
if (substr) return lowerStringSubstrCall(lowerer, call, receiver, argumentNodes);
588+
if (normalize) {
589+
const defaultForm = strLit("NFC", loc);
590+
const form = argumentNodes[0]
591+
? lowerOptionalArgument(lowerer, argumentNodes[0], STRING, defaultForm)
592+
: defaultForm;
593+
return { kind: "libCall", fn: "str.normalize", args: [receiver, form], type: STRING, loc };
594+
}
587595
if (concatDyn) {
588596
const spread = argumentNodes[0] as ts.SpreadElement;
589597
const values = lowerer.lowerExpr(spread.expression);

‎packages/compiler/src/frontend/lowering/lowerer.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ export interface LowerOptions {
357357
nativeStringSubstr?: boolean;
358358
/** Rust can concatenate a checked-dynamic spread argument vector. */
359359
nativeStringConcatDyn?: boolean;
360+
/** Rust provides Unicode String.prototype.normalize. */
361+
nativeStringNormalize?: boolean;
360362
/** --dynamic: the island engine is linked, so island constructs
361363
* (__island_eval) may lower. Off by default — without it they produce a
362364
* requires-dynamic diagnostic instead. */
@@ -459,6 +461,8 @@ export interface LowererMode {
459461
nativeStringSubstr?: boolean;
460462
/** Rust can concatenate a checked-dynamic spread argument vector. */
461463
nativeStringConcatDyn?: boolean;
464+
/** Rust provides Unicode String.prototype.normalize. */
465+
nativeStringNormalize?: boolean;
462466
/** Names of bodies a prior reachability pass reached; null lowers everything. */
463467
reachable?: ReadonlySet<string> | null;
464468
/** Coverage remainder: lower ONLY bodies outside `reachable`, skip the
@@ -572,6 +576,7 @@ export function lowerToIr(
572576
nativeDynamicSets: options.nativeDynamicSets ?? false,
573577
nativeStringSubstr: options.nativeStringSubstr ?? false,
574578
nativeStringConcatDyn: options.nativeStringConcatDyn ?? false,
579+
nativeStringNormalize: options.nativeStringNormalize ?? false,
575580
targetPlatform,
576581
startupCrash,
577582
ffiImports,
@@ -598,6 +603,7 @@ export function lowerToIr(
598603
nativeDynamicSets: options.nativeDynamicSets ?? false,
599604
nativeStringSubstr: options.nativeStringSubstr ?? false,
600605
nativeStringConcatDyn: options.nativeStringConcatDyn ?? false,
606+
nativeStringNormalize: options.nativeStringNormalize ?? false,
601607
targetPlatform,
602608
startupCrash,
603609
ffiImports,
@@ -632,6 +638,7 @@ export function lowerToIr(
632638
nativeDynamicSets: options.nativeDynamicSets ?? false,
633639
nativeStringSubstr: options.nativeStringSubstr ?? false,
634640
nativeStringConcatDyn: options.nativeStringConcatDyn ?? false,
641+
nativeStringNormalize: options.nativeStringNormalize ?? false,
635642
targetPlatform,
636643
startupCrash,
637644
ffiImports,
@@ -660,6 +667,7 @@ export function lowerToIr(
660667
nativeDynamicSets: options.nativeDynamicSets ?? false,
661668
nativeStringSubstr: options.nativeStringSubstr ?? false,
662669
nativeStringConcatDyn: options.nativeStringConcatDyn ?? false,
670+
nativeStringNormalize: options.nativeStringNormalize ?? false,
663671
targetPlatform,
664672
ffiImports,
665673
libraryCallbacks,
@@ -1077,6 +1085,7 @@ export class Lowerer {
10771085
readonly nativeDynamicSets: boolean;
10781086
readonly nativeStringSubstr: boolean;
10791087
readonly nativeStringConcatDyn: boolean;
1088+
readonly nativeStringNormalize: boolean;
10801089
readonly checker: ts.TypeChecker;
10811090
readonly diags: ScrDiagnostic[] = [];
10821091
readonly fnSigsBySymbol = new Map<ts.Symbol, FnSig>();
@@ -1848,6 +1857,7 @@ export class Lowerer {
18481857
this.nativeDynamicSets = mode.nativeDynamicSets ?? false;
18491858
this.nativeStringSubstr = mode.nativeStringSubstr ?? false;
18501859
this.nativeStringConcatDyn = mode.nativeStringConcatDyn ?? false;
1860+
this.nativeStringNormalize = mode.nativeStringNormalize ?? false;
18511861
this.reachable = mode.reachable ?? null;
18521862
this.remainder = mode.remainder ?? false;
18531863
this.alreadyFlushed = mode.alreadyFlushed ?? new Set();

‎packages/compiler/src/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -830,7 +830,7 @@ export function analyze(entryPath: string, opts: AnalyzeOptions = {}): AnalyzeRe
830830
const lowered = lowerWithFrontier(fe, {
831831
dynamic: opts.dynamic ?? false,
832832
coverage: true,
833-
statefulRegex: (opts.backend ?? "rust") === "rust", nativePromiseViews: (opts.backend ?? "rust") === "rust", nativeDenseArrays: (opts.backend ?? "rust") === "rust", nativeCollectionArrays: (opts.backend ?? "rust") === "rust", nativeClassDynamicRefs: (opts.backend ?? "rust") === "rust", nativeDynamicSets: (opts.backend ?? "rust") === "rust", nativeStringSubstr: (opts.backend ?? "rust") === "rust", nativeStringConcatDyn: (opts.backend ?? "rust") === "rust",
833+
statefulRegex: (opts.backend ?? "rust") === "rust", nativePromiseViews: (opts.backend ?? "rust") === "rust", nativeDenseArrays: (opts.backend ?? "rust") === "rust", nativeCollectionArrays: (opts.backend ?? "rust") === "rust", nativeClassDynamicRefs: (opts.backend ?? "rust") === "rust", nativeDynamicSets: (opts.backend ?? "rust") === "rust", nativeStringSubstr: (opts.backend ?? "rust") === "rust", nativeStringConcatDyn: (opts.backend ?? "rust") === "rust", nativeStringNormalize: (opts.backend ?? "rust") === "rust",
834834
targetPlatform: buildTargetPlatform(),
835835
...(ffi !== null ? { ffiImports: ffi.functions } : {}),
836836
});
@@ -1505,7 +1505,7 @@ async function compileTracked(
15051505
try {
15061506
lowered = lowerWithFrontier(fe, {
15071507
dynamic: opts.dynamic ?? false,
1508-
statefulRegex: (opts.backend ?? "rust") === "rust", nativePromiseViews: (opts.backend ?? "rust") === "rust", nativeDenseArrays: (opts.backend ?? "rust") === "rust", nativeCollectionArrays: (opts.backend ?? "rust") === "rust", nativeClassDynamicRefs: (opts.backend ?? "rust") === "rust", nativeDynamicSets: (opts.backend ?? "rust") === "rust", nativeStringSubstr: (opts.backend ?? "rust") === "rust", nativeStringConcatDyn: (opts.backend ?? "rust") === "rust",
1508+
statefulRegex: (opts.backend ?? "rust") === "rust", nativePromiseViews: (opts.backend ?? "rust") === "rust", nativeDenseArrays: (opts.backend ?? "rust") === "rust", nativeCollectionArrays: (opts.backend ?? "rust") === "rust", nativeClassDynamicRefs: (opts.backend ?? "rust") === "rust", nativeDynamicSets: (opts.backend ?? "rust") === "rust", nativeStringSubstr: (opts.backend ?? "rust") === "rust", nativeStringConcatDyn: (opts.backend ?? "rust") === "rust", nativeStringNormalize: (opts.backend ?? "rust") === "rust",
15091509
targetPlatform: buildPlatform,
15101510
...(ffi !== null ? { ffiImports: ffi.functions } : {}),
15111511
});

‎packages/compiler/src/ir/ir.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2205,6 +2205,8 @@ export type IrLibFn =
22052205
| "str.substr"
22062206
/** String.concat over a checked-dynamic spread argument vector (Rust). */
22072207
| "str.concatDyn"
2208+
/** Unicode String.prototype.normalize (Rust). */
2209+
| "str.normalize"
22082210
/** RegExp.escape (ES2025): per-code-point EncodeForRegExpEscape —
22092211
* leading ASCII alphanumeric hex-escapes, syntax characters and '/'
22102212
* take a backslash, other punctuators/whitespace/line terminators
@@ -7798,6 +7800,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
77987800
// decodeURIComponent throws the spec's URIError on bad hex/invalid
77997801
// UTF-8 octets (encodeURIComponent never throws — see the IrLibFn doc).
78007802
"str.decodeUriComponent",
7803+
"str.normalize",
78017804
// The base64 globals: atob/btoa throw the catchable DOMException
78027805
// InvalidCharacterError on malformed input; the zero-argument form
78037806
// always throws Node's TypeError [ERR_MISSING_ARGS].

‎packages/compiler/src/ir/validate.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ export const LIB_FN_SIGS: Record<IrLibFn, { argTypes: (IrType | null)[]; result:
261261
"str.encodeUriComponent": { argTypes: [STRING], result: STRING },
262262
"str.substr": { argTypes: [STRING, F64, F64], result: STRING },
263263
"str.concatDyn": { argTypes: [STRING, DYN], result: STRING },
264+
"str.normalize": { argTypes: [STRING, STRING], result: STRING },
264265
// The base64 globals: the argument is a dyn value (WebIDL ToString
265266
// runs in the runtime); the zero-argument form always throws.
266267
"str.atob": { argTypes: [DYN], result: STRING },

‎packages/runtime-rust/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/runtime-rust/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ rustls = { version = "=0.23.43", default-features = false, features = ["ring", "
4040
rustls-pemfile = "=2.2.0"
4141
signal-hook = { version = "=0.4.4", default-features = false }
4242
url = "=2.5.0"
43+
unicode-normalization = "=0.1.25"
4344
webpki-roots = "=1.0.9"
4445

4546
[target.'cfg(all(not(windows), not(target_os = "wasi")))'.dependencies]

‎packages/runtime-rust/src/strings_and_process.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use unicode_normalization::UnicodeNormalization;
2+
13
fn array_index(index: f64, allow_end: bool, len: usize) -> usize {
24
if !index.is_finite() || index < 0.0 || index.fract() != 0.0 || index > usize::MAX as f64 {
35
panic!("scriptc: invalid array index");
@@ -418,6 +420,42 @@ pub fn string_substr(value: &JsString, start: f64, length: f64) -> JsString {
418420
string_from_utf16(&units[start..end])
419421
}
420422

423+
pub fn string_normalize(value: &JsString, form: &JsString) -> JsString {
424+
let Some(form) = form.well_formed_utf8() else {
425+
throw_range_error("The normalization form should be one of NFC, NFD, NFKC, NFKD.".to_owned());
426+
};
427+
if !matches!(form, "NFC" | "NFD" | "NFKC" | "NFKD") {
428+
throw_range_error("The normalization form should be one of NFC, NFD, NFKC, NFKD.".to_owned());
429+
}
430+
let mut output = JsStringBuilder::new();
431+
let mut run = String::new();
432+
let flush = |run: &mut String, output: &mut JsStringBuilder| {
433+
if run.is_empty() {
434+
return;
435+
}
436+
let normalized = match form {
437+
"NFC" => run.nfc().collect::<String>(),
438+
"NFD" => run.nfd().collect::<String>(),
439+
"NFKC" => run.nfkc().collect::<String>(),
440+
"NFKD" => run.nfkd().collect::<String>(),
441+
_ => unreachable!(),
442+
};
443+
output.push_str(&normalized);
444+
run.clear();
445+
};
446+
for item in char::decode_utf16(value.encode_utf16()) {
447+
match item {
448+
Ok(ch) => run.push(ch),
449+
Err(error) => {
450+
flush(&mut run, &mut output);
451+
output.push_unit(error.unpaired_surrogate());
452+
}
453+
}
454+
}
455+
flush(&mut run, &mut output);
456+
output.finish()
457+
}
458+
421459
pub fn string_slice(value: &JsString, start: f64, end: f64) -> JsString {
422460
let units: Vec<u16> = value.encode_utf16().collect();
423461
let start = relative_string_index(start, units.len());

0 commit comments

Comments
 (0)