diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a41130f1..017db1f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -362,28 +362,33 @@ jobs: with: shared-key: memory-gate - name: Install redis tools - # Hardened after three hangs-to-timeout on hosted runners (2026-08-19): + # Hardened after repeated hangs-to-timeout on hosted runners (2026-08-19): # force IPv4 (azure apt mirrors black-hole IPv6), bound every network - # operation, and retry — a hung apt must fail THIS step in minutes, - # not cancel the whole run at the workflow timeout. - timeout-minutes: 5 + # operation, retry, and — crucially — try the install from the runner's + # PRE-BAKED apt lists BEFORE any network `update`. `apt-get update` was + # observed timing out at 120s repeatedly during an apt-mirror incident; + # redis-tools/jq are already indexed on the ubuntu-latest image, so a + # cache-first install succeeds without ever touching the failing mirror. + # timeout-minutes must exceed 3 full attempts (install+update+install) so + # the retry loop is not cut off mid-flight (the old 5m killed attempt 3). + timeout-minutes: 9 run: | # set +e is load-bearing: GitHub's default `shell: bash` is `bash -e`, # so the FIRST `apt-get` non-zero (e.g. `timeout` exit 124) aborts the # whole step instantly -- the retry loop below never reaches attempt 2. - # Observed 2026-08-19: attempt-1 update timed out, step exited 124, no - # retry. Disable -e here so a failed attempt falls through to the next. set +e # DPkg::Lock::Timeout: fresh runners often hold the dpkg frontend # lock via unattended-upgrades; without it apt waits FOREVER with no - # output (observed: 5-min step timeout, zero log lines). The outer - # `timeout` bounds each attempt against any other silent stall. + # output. The outer `timeout` bounds each op against any silent stall. APT_OPTS="-o Acquire::ForceIPv4=true -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=60" + install() { sudo timeout 120 apt-get $APT_OPTS install -y -qq redis-tools jq; } for i in 1 2 3; do echo "apt attempt $i" - sudo timeout 120 apt-get $APT_OPTS update -qq \ - && sudo timeout 120 apt-get $APT_OPTS install -y -qq redis-tools jq \ - && { echo "apt attempt $i ok"; exit 0; } + # 1) cache-first: no network `update`, works during a mirror outage. + install && { echo "apt attempt $i ok (cached lists)"; exit 0; } + # 2) fall back to refreshing lists, then install again. + sudo timeout 120 apt-get $APT_OPTS update -qq + install && { echo "apt attempt $i ok (after update)"; exit 0; } echo "apt attempt $i failed; retrying" >&2 sleep 15 done diff --git a/CHANGELOG.md b/CHANGELOG.md index edd40907..12785844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security +- **ACL `~pattern` restrictions were silently unenforced for most multi-key commands** (#566). + `AclTable::check_key_permission` read a command's keys from `extract_command_keys`, a + hand-maintained match on the command name whose fallthrough returned an EMPTY key list — and an + empty key list is not "checked less precisely", it makes the permission loop a no-op, so every + `~pattern` was ignored outright for any command the list forgot. Measured against a live server, + a user restricted to `~app:*` reached arbitrary keys through 21 distinct commands, in both + `--shards 1` and `--shards 4`: `COPY`, `ZRANGESTORE` (both positions), `SMOVE`'s DESTINATION (it + was listed, but as a single-key command), `LMPOP`/`ZMPOP`/`BLMPOP`/`BZMPOP`, `SINTERCARD`, + `ZDIFF`/`ZINTER`/`ZUNION`/`ZINTERCARD`, `SORT ... STORE`, `SORT ... BY `, + `GEORADIUS ... STORE`, `EVAL`'s declared keys and `MEMORY USAGE`. + Key extraction is now **derived from the command registry's key specs** + (`COMMAND_META` `first_key`/`last_key`/`step`), so a command that declares its keys is enforced + automatically; hand-written arms remain only for layouts a fixed spec cannot express (`numkeys` + vectors, positional `STORE` clauses, the `STREAMS` token, subcommand-shaped key positions). + Extraction **fails closed**: a command that names keys but whose argv cannot be enumerated — or a + command missing from the registry entirely — is DENIED with the standard `NOPERM` error and + logged once per command name, so the next command that ships without a key spec fails safe + instead of falling open. `SORT`'s `BY`/`GET` patterns read key names computed at runtime and are + therefore refused for key-restricted users (`BY nosort` / `GET #` are unaffected). Commands that + genuinely name no key (`PING`, `CONFIG`, `SUBSCRIBE`, `KEYS`, the `FT.*`/`GRAPH.*` families, ...) + are unaffected, and a registry sweep test now fails if a NEW command declares no keys without + being reviewed. Unrestricted and `~*` users still short-circuit before any extraction; the new + path borrows key slices into a `SmallVec` and no longer heap-allocates per command. + ### Fixed - **Blocking pops queued inside `MULTI` answer the wrong reply SHAPE** (#524). `BLPOP`/`BRPOP`/ `BZPOPMIN`/`BZPOPMAX` were rewritten at queue time into `LPOP`/`RPOP`/`ZPOPMIN`/`ZPOPMAX`, whose diff --git a/src/acl/keyspec.rs b/src/acl/keyspec.rs new file mode 100644 index 00000000..22503128 --- /dev/null +++ b/src/acl/keyspec.rs @@ -0,0 +1,1162 @@ +//! Key extraction for ACL `~pattern` enforcement — derived from the command +//! registry's key specs, fail-closed on anything it cannot enumerate. +//! +//! # Why this module exists (moon#566) +//! +//! `AclTable::check_key_permission` used to get a command's keys from a +//! hand-maintained `match` on the command name whose fallthrough returned an +//! empty vector. An empty key list is **not** "checked less precisely": the +//! permission loop simply never runs, so every `~pattern` was silently +//! ignored for any command the list forgot. `SMOVE` (listed, but as a +//! single-key command, leaving its DESTINATION unchecked), `COPY`, +//! `ZRANGESTORE`, `LMPOP`/`ZMPOP`/`BLMPOP`/`BZMPOP`, `SINTERCARD`, +//! `ZDIFF`/`ZINTER`/`ZUNION`, `SORT ... STORE` and `GEORADIUS ... STORE` all +//! had that shape: a `~cache:*` user could read from — and write to — +//! arbitrary keys through them. +//! +//! The list is now DERIVED from [`crate::command::metadata::COMMAND_META`] +//! (`first_key`/`last_key`/`step`, Redis argv semantics), so a command that +//! declares its keys there is enforced automatically. The hand-written arms +//! that remain cover only the layouts a fixed key spec provably cannot +//! express — `numkeys`-counted key vectors, positional `STORE` clauses, the +//! `STREAMS` token, and subcommand-shaped key positions. +//! +//! # Fail-closed contract +//! +//! [`command_keys`] answers one of three things: +//! +//! * [`CommandKeys::None`] — the command provably names NO key (`PING`, +//! `CONFIG`, `SUBSCRIBE`, ...). The caller must not deny on key patterns. +//! * [`CommandKeys::Keys`] — the exact keys this invocation touches. +//! * [`CommandKeys::Indeterminate`] — the command is known (or suspected) to +//! touch keys, but THIS argv could not be enumerated. The caller must DENY. +//! A future command that ships without a key spec therefore fails safe +//! instead of falling open. +//! +//! # Hot path +//! +//! `check_key_permission` short-circuits for unrestricted users and for `~*` +//! BEFORE calling in here, so this code runs only for key-restricted users. +//! Even so it is allocation-free for up to four keys (`SmallVec` inline +//! capacity) and borrows key bytes out of the argv rather than copying them — +//! strictly cheaper than the `Vec<&[u8]>` it replaces, which heap-allocated +//! for every single-key command. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use parking_lot::Mutex; +use smallvec::SmallVec; + +use crate::command::metadata; +use crate::protocol::Frame; + +/// Borrowed key slices. Four inline slots cover every fixed-arity keyed +/// command in the registry; only variadic forms (`DEL a b c d e`) spill. +pub(crate) type KeyVec<'a> = SmallVec<[&'a [u8]; 4]>; + +/// Outcome of key extraction. See the module docs for the contract. +#[derive(Debug)] +pub(crate) enum CommandKeys<'a> { + /// The command provably names no key. + None, + /// The keys this invocation touches (never empty). + Keys(KeyVec<'a>), + /// Keys exist (or may exist) but could not be enumerated — deny. + Indeterminate, +} + +/// Commands that are dispatched but carry no entry in `COMMAND_META`, and +/// genuinely name no key. Without this list they would be `Indeterminate` +/// (fail-closed) and a key-restricted user would lose them. +/// +/// Keep this in sync with the dispatch table: a NEW keyless command added +/// outside `COMMAND_META` must be listed here, and a new KEYED command must +/// get a `COMMAND_META` entry (the point of the fail-closed default is that +/// forgetting is safe). +const UNREGISTERED_KEYLESS: &[&[u8]] = &[ + b"HEALTHZ", + b"READYZ", + b"PUBSUB", + b"ASKING", + b"READONLY", + b"READWRITE", +]; + +/// Extract the keys `cmd`/`args` touch. `args` EXCLUDES the command name, so +/// key-spec index `N` maps to `args[N - 1]`. +pub(crate) fn command_keys<'a>(cmd: &[u8], args: &'a [Frame]) -> CommandKeys<'a> { + // Layouts a fixed first/last/step spec cannot express come first: some of + // them (SORT, OBJECT, ZUNIONSTORE) DO have a spec, but it describes only + // part of the truth. + if let Some(keys) = movable_keys(cmd, args) { + return keys; + } + + let Some(meta) = metadata::lookup(cmd) else { + // FT.* / GRAPH.* address indexes and graphs, not keyspace keys; ACL + // key patterns do not cover that namespace at all (tracked + // separately). Everything else unknown fails closed. + if cmd.len() > 3 && cmd[..3].eq_ignore_ascii_case(b"FT.") + || cmd.len() > 6 && cmd[..6].eq_ignore_ascii_case(b"GRAPH.") + || UNREGISTERED_KEYLESS + .iter() + .any(|k| cmd.eq_ignore_ascii_case(k)) + { + return CommandKeys::None; + } + return CommandKeys::Indeterminate; + }; + + if meta.first_key <= 0 { + return CommandKeys::None; + } + + let argc = args.len(); + let first = meta.first_key as usize; + // Redis argv semantics: index 1 is the first argument after the command + // name, so the last valid index is `argc`. A negative `last_key` counts + // back from there (-1 = last argument, -2 = second to last). + let last = if meta.last_key < 0 { + let back = (-meta.last_key) as usize; + match (argc + 1).checked_sub(back) { + Some(idx) => idx, + None => return CommandKeys::Indeterminate, + } + } else { + meta.last_key as usize + }; + // A declared key position that the argv does not reach is a malformed + // invocation — deny rather than silently check fewer keys. + if first > last || last > argc { + return CommandKeys::Indeterminate; + } + + let step = if meta.step > 0 { meta.step as usize } else { 1 }; + let mut keys = KeyVec::new(); + let mut i = first; + while i <= last { + match key_bytes(&args[i - 1]) { + Some(k) => keys.push(k), + None => return CommandKeys::Indeterminate, + } + i += step; + } + if keys.is_empty() { + return CommandKeys::Indeterminate; + } + CommandKeys::Keys(keys) +} + +/// Log ONCE per command name that a key check failed closed, so an operator +/// can tell a real permission denial from a missing key spec. +/// +/// Bounded: an attacker can otherwise mint unbounded command names. Past the +/// cap the denial still happens, only the log line is dropped. +pub(crate) fn warn_indeterminate(cmd: &[u8]) { + const CAP: usize = 128; + static WARNED: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let name = String::from_utf8_lossy(cmd).to_ascii_uppercase(); + let mut seen = WARNED.lock(); + if seen.len() >= CAP || seen.contains(name.as_str()) { + return; + } + seen.insert(name.clone().into_boxed_str()); + drop(seen); + tracing::warn!( + command = %name, + "ACL denied a key-restricted user: this command's key arguments could not be \ + determined, so `~pattern` cannot be enforced for it (fail-closed). Add a \ + COMMAND_META key spec, or a movable-key arm in acl::keyspec, to allow it." + ); +} + +#[inline] +fn key_bytes(frame: &Frame) -> Option<&[u8]> { + match frame { + Frame::BulkString(b) | Frame::SimpleString(b) => Some(b.as_ref()), + _ => None, + } +} + +/// Key layouts that `first_key`/`last_key`/`step` cannot express. +/// +/// Returns `None` when the command is not one of them (the caller then uses +/// the meta-derived walk). Matched on `(len, first byte)` first so the common +/// single-key commands fall through after one integer compare. +fn movable_keys<'a>(cmd: &[u8], args: &'a [Frame]) -> Option> { + let len = cmd.len(); + if len == 0 { + return None; + } + let b0 = cmd[0] | 0x20; + let keys = match (len, b0) { + // ---- numkeys-counted key vectors ---- + // numkeys key [key ...] ... + (5, b'l') if cmd.eq_ignore_ascii_case(b"LMPOP") => numkeys_keys(args, 0, false), + (5, b'z') if cmd.eq_ignore_ascii_case(b"ZMPOP") => numkeys_keys(args, 0, false), + (5, b'z') if cmd.eq_ignore_ascii_case(b"ZDIFF") => numkeys_keys(args, 0, false), + (6, b'z') if cmd.eq_ignore_ascii_case(b"ZINTER") || cmd.eq_ignore_ascii_case(b"ZUNION") => { + numkeys_keys(args, 0, false) + } + (10, b'z') if cmd.eq_ignore_ascii_case(b"ZINTERCARD") => numkeys_keys(args, 0, false), + (10, b's') if cmd.eq_ignore_ascii_case(b"SINTERCARD") => numkeys_keys(args, 0, false), + // timeout numkeys key [key ...] + (6, b'b') if cmd.eq_ignore_ascii_case(b"BLMPOP") || cmd.eq_ignore_ascii_case(b"BZMPOP") => { + numkeys_keys(args, 1, false) + } + // script|sha|function numkeys key [key ...] [arg ...] + (4, b'e') if cmd.eq_ignore_ascii_case(b"EVAL") => numkeys_keys(args, 1, false), + (7, b'e') if cmd.eq_ignore_ascii_case(b"EVALSHA") => numkeys_keys(args, 1, false), + (5, b'f') if cmd.eq_ignore_ascii_case(b"FCALL") => numkeys_keys(args, 1, false), + (8, b'f') if cmd.eq_ignore_ascii_case(b"FCALL_RO") => numkeys_keys(args, 1, false), + // dest numkeys key [key ...] [WEIGHTS ...] [AGGREGATE ...] + // The registry spec names only `dest` (first_key == last_key == 1). + (11, b'z') + if cmd.eq_ignore_ascii_case(b"ZUNIONSTORE") + || cmd.eq_ignore_ascii_case(b"ZINTERSTORE") => + { + numkeys_keys(args, 1, true) + } + (10, b'z') if cmd.eq_ignore_ascii_case(b"ZDIFFSTORE") => numkeys_keys(args, 1, true), + + // ---- positional STORE clauses (source key + optional destination) ---- + (4, b's') if cmd.eq_ignore_ascii_case(b"SORT") => source_plus_store(args, true), + (7, b's') if cmd.eq_ignore_ascii_case(b"SORT_RO") => source_plus_store(args, true), + (9, b'g') if cmd.eq_ignore_ascii_case(b"GEORADIUS") => source_plus_store(args, false), + (17, b'g') if cmd.eq_ignore_ascii_case(b"GEORADIUSBYMEMBER") => { + source_plus_store(args, false) + } + + // ---- two-key move with no registry entry ---- + // `RPOPLPUSH src dst` is dispatched by moon#520's work but carries no + // COMMAND_META entry, so the meta-derived walk cannot see its keys and + // the fail-closed default would refuse it outright for key-restricted + // users. Its blocking twin BRPOPLPUSH is in the registry (1..2) and + // needs no arm here. Delete this arm once RPOPLPUSH gets a key spec. + (9, b'r') if cmd.eq_ignore_ascii_case(b"RPOPLPUSH") => two_keys(args), + + // ---- keys after the STREAMS token ---- + (5, b'x') if cmd.eq_ignore_ascii_case(b"XREAD") => stream_keys(args), + (10, b'x') if cmd.eq_ignore_ascii_case(b"XREADGROUP") => stream_keys(args), + + // ---- subcommand first, key second ---- + (6, b'o') if cmd.eq_ignore_ascii_case(b"OBJECT") => subcommand_key(args), + (5, b'x') if cmd.eq_ignore_ascii_case(b"XINFO") => subcommand_key(args), + (6, b'm') if cmd.eq_ignore_ascii_case(b"MEMORY") => subcommand_key(args), + (6, b'x') if cmd.eq_ignore_ascii_case(b"XGROUP") => subcommand_key(args), + + _ => return None, + }; + Some(keys) +} + +/// `... numkeys key [key ...]` with `numkeys` at `nk_idx`, plus an optional +/// destination key at `args[0]` (the `Z*STORE` family). +fn numkeys_keys(args: &[Frame], nk_idx: usize, dest_at_zero: bool) -> CommandKeys<'_> { + let Some(nk) = args + .get(nk_idx) + .and_then(key_bytes) + .and_then(|b| std::str::from_utf8(b).ok()) + .and_then(|s| s.parse::().ok()) + else { + return CommandKeys::Indeterminate; + }; + let first = nk_idx + 1; + // `numkeys` larger than the argv is malformed; the command errors out + // anyway, and enumerating fewer keys than declared must not silently + // reduce enforcement. `checked_add` is load-bearing, not defensive + // decoration: `nk` is attacker-controlled, so `first + nk` can overflow + // `usize` — in release (no overflow-checks) it WRAPS, the `<` guard then + // sees a tiny sum and the slice below panics `&args[1..0]`. A panic here + // is reachable by any key-restricted user = remote DoS inside ACL. + let Some(end) = first.checked_add(nk).filter(|&e| e <= args.len()) else { + return CommandKeys::Indeterminate; + }; + let mut keys = KeyVec::new(); + if dest_at_zero { + match args.first().and_then(key_bytes) { + Some(dest) => keys.push(dest), + None => return CommandKeys::Indeterminate, + } + } + for frame in &args[first..end] { + match key_bytes(frame) { + Some(k) => keys.push(k), + None => return CommandKeys::Indeterminate, + } + } + if keys.is_empty() { + // `EVAL