diff --git a/README.md b/README.md index 92036350..26a10a19 100644 --- a/README.md +++ b/README.md @@ -120,8 +120,13 @@ cargo test ### New extensions * Unicode characters can be specified in regular expression pattern, replacement and transliteration sequences using `\uXXXX` or `\UXXXXXXXX` sequences. + +### Incompatible extensions +The `-U` or `--uutil-extensions` option enables useful extensions or bug fixes +that aren't compatible with GNU sed or POSIX. + * The `l` command lists Unicode characters using the `\uXXXX` and `\UXXXXXXXX` - sequences. + escapes rather than as octal UTF-8 byte sequences. ### Incompatibilities * Similarly to GNU _sed_, input is processed as raw bytes or as valid UTF-8 diff --git a/src/sed/command.rs b/src/sed/command.rs index 30a47be1..43dedbb7 100644 --- a/src/sed/command.rs +++ b/src/sed/command.rs @@ -38,6 +38,7 @@ pub struct ProcessingContext { pub sandbox: bool, pub unbuffered: bool, pub null_data: bool, + pub uutil_extensions: bool, // Other context /// Currently processed input file name (not script) diff --git a/src/sed/mod.rs b/src/sed/mod.rs index 7e1b0b68..1ed79a56 100644 --- a/src/sed/mod.rs +++ b/src/sed/mod.rs @@ -114,6 +114,11 @@ pub fn uu_app() -> Command { arg!(-s --separate "Consider files as separate rather than as a long stream."), arg!(--sandbox "Operate in a sandbox by disabling e/r/w commands."), arg!(-u --unbuffered "Load minimal input data and flush output buffers regularly."), + Arg::new("uutil-extensions") + .short('U') + .long("uutil-extensions") + .help("Enable incompatible extensions.") + .action(clap::ArgAction::SetTrue), Arg::new("null-data") .short('z') .long("null-data") @@ -243,6 +248,7 @@ fn build_context(matches: &ArgMatches) -> UResult { sandbox: matches.get_flag("sandbox"), unbuffered: matches.get_flag("unbuffered"), null_data: matches.get_flag("null-data"), + uutil_extensions: matches.get_flag("uutil-extensions"), // Environment character_mode: character_mode_for_locale(&locale)?, @@ -389,6 +395,7 @@ mod tests { assert!(!ctx.sandbox); assert!(!ctx.unbuffered); assert!(!ctx.null_data); + assert!(!ctx.uutil_extensions); } #[test] @@ -406,6 +413,7 @@ mod tests { "-s", "--sandbox", "-u", + "-U", "-z", ]); @@ -424,6 +432,7 @@ mod tests { assert!(ctx.sandbox); assert!(ctx.unbuffered); assert!(ctx.null_data); + assert!(ctx.uutil_extensions); } #[test] diff --git a/src/sed/processor.rs b/src/sed/processor.rs index 6549b9f2..10a97188 100644 --- a/src/sed/processor.rs +++ b/src/sed/processor.rs @@ -573,7 +573,8 @@ fn list( let mut list_line = ListLine::new(max_width); - if context.character_mode == CharacterMode::Byte { + if !context.uutil_extensions || context.character_mode == CharacterMode::Byte { + // List non-ASCII bytes in octal. for &byte in line.as_bytes() { if byte == b'\n' { list_line.write_embedded_newline(output)?; @@ -583,6 +584,7 @@ fn list( list_line.write_item(output, &out_str)?; } } else { + // List non-ASCII 8-bit characters in octal; Unicode in hex \u or \U. let line = line.as_str().map_err(|e| { input_runtime_error::<()>( location, diff --git a/tests/by-util/test_sed.rs b/tests/by-util/test_sed.rs index 01fbbd78..293acc64 100644 --- a/tests/by-util/test_sed.rs +++ b/tests/by-util/test_sed.rs @@ -1776,16 +1776,31 @@ fn filename_non_posix() { .stderr_contains("invalid command code"); } -/// List Unicode input under an explicit UTF-8 locale. +/// List Unicode input under an explicit UTF-8 locale +/// with uutil extensions enabled. #[test] fn list_unicode() { new_ucmd!() .env("LC_ALL", "C.UTF-8") - .args(&["l 60", "input/unicode"]) + .args(&["--uutil-extensions", "l 60", "input/unicode"]) .succeeds() .stdout_is_fixture_bytes("output/list_unicode"); } +// List Unicode input without uutil extensions should generate octal bytes. +check_output!(list_unicode_octal, ["l 60", "input/unicode"]); + +/// List Unicode input without uutil extensions should generate octal bytes, +/// even under an explicit UTF-8 locale. +#[test] +fn list_unicode_octal_env() { + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["l 60", "input/unicode"]) + .succeeds() + .stdout_is_fixture_bytes("output/list_unicode_octal"); +} + /// List invalid UTF-8 bytes without decoding in byte mode. #[test] fn list_invalid_utf8_byte_locale() { diff --git a/tests/fixtures/sed/output/list_unicode_octal b/tests/fixtures/sed/output/list_unicode_octal new file mode 100644 index 00000000..1e47bff9 --- /dev/null +++ b/tests/fixtures/sed/output/list_unicode_octal @@ -0,0 +1,5 @@ +Hello World or \316\232\316\261\316\273\316\267\316\274\316\ +\255\317\201\316\261 \316\272\317\214\317\203\316\274\316\ +\265 or \343\201\223\343\202\223\343\201\253\343\201\241\ +\343\201\257 \344\270\226\347\225\214 \360\237\230\200$ +Hello World or Καλημέρα κόσμε or こんにちは 世界 😀