feat(info): ten INFO fields from real sources, three waived with reasons (EC9)

.github/workflows/ci.yml

@@ -421,9 +421,11 @@ jobs:
         run: ./scripts/test-client-compat.sh --filter __none__ --info-manifest --record tmp/client-compat-info.json
         env:
           MOON_BIN: ${{ env.CARGO_TARGET_DIR }}/release/moon
-        # Reports the missing-field set; not yet a gate — info-observability
-        # owns closing it, and this flips to a hard gate when that task lands.
-        continue-on-error: true
+        # HARD GATE as of EC9. Every pinned field is either emitted from a real
+        # source or waived in info_fields.txt with a recorded reason, and the
+        # harness refuses an unreasoned waiver (exit 2) as well as a waiver on
+        # a field Moon has since started emitting — so the waiver list cannot
+        # silently go stale the way the registry sweep's did.
       # ── SDK wire-form guards ────────────────────────────────────────────
       # The SDK tree had no CI of any kind, which is how five helpers shipped
       # sending wire forms the server rejects on every call. They live here

.gitignore

@@ -106,6 +106,8 @@ tmp/
 /target-check-tokio/
 /target-check-monoio/
 /target-check/
+/target-fu/
+/target-rf/
 target-fast/
 # CARGO_TARGET_DIR used for the local release-fast build+test loop
 /target-rf/

CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- **Nine INFO fields a standard monitoring stack reads.** `tcp_port`, `uptime_in_seconds`,
+  `uptime_in_days`, `aof_last_write_status`, `aof_last_bgrewrite_status`,
+  `rdb_changes_since_last_save`, `sync_full`, `sync_partial_ok` and `sync_partial_err` are now
+  emitted, each from a real source rather than a constant: uptime from a start instant captured
+  before the listener binds, `rdb_changes_since_last_save` from a sharded
+  keyspace-mutation counter reset at save completion, and the three `sync_*` counters recorded at
+  the one point in the PSYNC handshake where full-vs-partial is still distinguishable (`PSYNC ? -1`
+  counts as a full resync the replica ASKED for, not a partial that failed). `tcp_port` reports the
+  configured listener port, not the port the INFO connection arrived on — behind a container port
+  map the two differ, and the field exists so a client can hand a peer a reachable address.
+  The four fields Moon cannot answer truthfully are recorded as waivers with reasons instead of
+  being emitted: `latest_fork_usec` (Moon never calls `fork(2)`; BGSAVE snapshots in-process), the
+  two `client_recent_max_*_buffer` high-water marks (untracked, and tracking them means a counter on
+  every connection read and write), and `used_memory_lua` — which was implemented and then withdrawn
+  when measurement showed the value the shard can publish is ~2 orders of magnitude below the real
+  VM footprint on the shipped monoio runtime (80 bytes against tokio's 26183, same `setup_lua_vm`,
+  ruled out as Cargo feature unification). This keeps the INFO emitter's standing rule intact: a
+  wrong number on a dashboard is worse than an absent one.
+
+### Changed
+- **The `INFO field coverage` CI step is now a hard gate**, no longer `continue-on-error`. The
+  pinned-field harness gained a waiver syntax (`field  # WAIVED: <reason>`) that refuses an
+  unreasoned waiver at load time (exit 2) and reports a waiver on a field Moon has since started
+  emitting as a failure — so the list cannot go stale unnoticed the way the registry sweep's did.
+
 - **`CLUSTER SHARDS`, `CLUSTER MYSHARDID`, `READONLY` and `READWRITE`** — the four verbs a
   cluster-aware client needs to bootstrap. `CLUSTER SHARDS` reports every shard cluster-wide, one
   entry per master with its replicas, master first; a shard that has lost every live node reports an

scripts/client-compat/differ.py

@@ -612,9 +612,47 @@ def _compare_entry(self, entry: Entry, protocol: str, context: str,
         return Result(entry.name, protocol, context, r_sent, r_raw, m_raw,
                       verdict, v.divergence, v.detail, entry.expect_diff)
 
+    @staticmethod
+    def _parse_info_manifest(path: str) -> list[tuple[str, str | None]]:
+        """Pinned INFO fields as (name, waiver_reason | None).
+
+        A line may carry an inline waiver:
+
+            latest_fork_usec  # WAIVED: Moon never calls fork(2)
+
+        A waiver is for a field Moon cannot answer TRUTHFULLY -- the INFO
+        emitter's standing rule is that such a field is omitted rather than
+        reported as a constant, because a hardcoded zero is indistinguishable
+        from a healthy server on a dashboard. The waiver records why, so the
+        omission is a decision on the record instead of a gap.
+
+        A waiver with no reason is rejected at load: an unexplained waiver is
+        how a real gap gets parked forever.
+        """
+        out: list[tuple[str, str | None]] = []
+        with open(path) as f:
+            for lineno, ln in enumerate(f, 1):
+                ln = ln.strip()
+                if not ln or ln.startswith("#"):
+                    continue
+                name, _, comment = ln.partition("#")
+                name = name.strip()
+                comment = comment.strip()
+                waiver = None
+                if comment.upper().startswith("WAIVED"):
+                    reason = comment.partition(":")[2].strip()
+                    if not reason:
+                        raise HarnessError(
+                            "ERR_UNREASONED_WAIVER",
+                            f"{path}:{lineno}: '{name}' is waived with no reason. "
+                            f"Write `{name}  # WAIVED: <why moon cannot answer "
+                            f"this truthfully>`.")
+                    waiver = reason
+                out.append((name, waiver))
+        return out
+
     def _info_coverage(self, rport: int, mport: int) -> list[Result]:
-        with open(self.cfg.info_manifest) as f:
-            fields = [ln.strip() for ln in f if ln.strip() and not ln.startswith("#")]
+        fields = self._parse_info_manifest(self.cfg.info_manifest)
         rc, mc = RespConn(rport, "resp2"), RespConn(mport, "resp2")
         try:
             sent = encode_command(["INFO"])
@@ -628,10 +666,29 @@ def _info_coverage(self, rport: int, mport: int) -> list[Result]:
         moon_body = (m_node.value or b"").decode("latin1") if m_node else ""
         redis_body = (r_node.value or b"").decode("latin1") if r_node else ""
         out = []
-        for fname in fields:
+        for fname, waiver in fields:
             pat = rf"^{re.escape(fname)}:"
             in_moon = re.search(pat, moon_body, re.M) is not None
             in_redis = re.search(pat, redis_body, re.M) is not None
+            if waiver is not None:
+                if in_moon:
+                    # Self-invalidating waiver: the field SHIPPED. Leaving the
+                    # waiver in place would keep the manifest green over a
+                    # surface it had stopped needing to excuse -- exactly the
+                    # stale-waiver failure mode the registry sweep hit.
+                    verdict, div = "diff", "value"
+                    detail = (f"'{fname}' is waived ({waiver}) but moon now "
+                              f"EMITS it — delete the waiver, the gap is closed")
+                    out.append(Result(f"info:{fname}", "resp2", "standalone",
+                                      sent, r_raw, m_raw, verdict, div, detail))
+                    continue
+                # Carry the reason on `waiver_reason`, the field the reporter
+                # and the JSON record both read — a waiver whose reason prints
+                # as `None` is indistinguishable from an unreasoned one.
+                out.append(Result(f"info:{fname}", "resp2", "standalone", sent,
+                                  r_raw, m_raw, "waived", None,
+                                  f"waived: {waiver}", waiver_reason=waiver))
+                continue
             if not in_redis:
                 # The oracle does not emit it either: the pinned list is wrong,
                 # not Moon. Reporting this as a Moon defect would manufacture a

scripts/client-compat/info_fields.txt

@@ -19,14 +19,14 @@ uptime_in_days
 # clients
 connected_clients
 blocked_clients
-client_recent_max_input_buffer
-client_recent_max_output_buffer
+client_recent_max_input_buffer  # WAIVED: per-client input/output buffer high-water marks are not tracked. CLIENT LIST reports qbuf=0 for the same reason. Sampling them means touching a counter on every read and write in the connection hot path, and a hardcoded 0 here would tell an operator hunting a buffer blowup that no client has ever buffered anything.
+client_recent_max_output_buffer  # WAIVED: see client_recent_max_input_buffer — same untracked source, same hot-path cost.
 
 # memory
 used_memory
 used_memory_rss
 used_memory_peak
-used_memory_lua
+used_memory_lua  # WAIVED: the per-shard `mlua` VM's used_memory() is reachable, but on the SHIPPED monoio runtime the value the shard can publish is ~2 orders of magnitude below the real footprint (80 bytes for a VM that reports 26183 under tokio, same setup_lua_vm, same mlua features -- measured both ways, and ruled out as feature unification by building tokio WITH graph+text-index). Emitting it would put a number on a dashboard that is wrong on the runtime that ships. Tracked separately; unwaive when the monoio path samples the VM that actually executes scripts.
 maxmemory
 maxmemory_policy
 mem_fragmentation_ratio
@@ -58,7 +58,7 @@ keyspace_hits
 keyspace_misses
 pubsub_channels
 pubsub_patterns
-latest_fork_usec
+latest_fork_usec  # WAIVED: Moon never calls fork(2). BGSAVE snapshots in-process via a copy-on-write epoch cursor, so there is no fork to time. Redis's own value is a fork-duration measurement; emitting 0 would read as 'forks are instant' rather than 'this server does not fork'.
 
 # replication
 role

src/admin/metrics_setup.rs

@@ -188,6 +188,100 @@ fn total_commands_sum() -> u64 {
         .sum()
 }
 
+// ── EC9: keyspace-change counter behind `rdb_changes_since_last_save` ────
+// The "is a save worth doing" signal every backup script reads. Sharded over
+// the same padded per-thread slots as the total-commands counter for the same
+// reason (a single line would bounce across every shard core at write rate);
+// the increment is one relaxed fetch_add on the thread's own line.
+//
+// Counted at the storage funnels (`Database::set` / `remove` / `get_mut` /
+// `clear` / `set_expiry`), not at dispatch: dispatch does not know whether a
+// command mutated, and a phf flags lookup on the hot path is exactly the cost
+// this codebase's perf invariants forbid. `get_mut` hands out mutable access
+// that the caller may or may not use, so the count can run slightly HIGH.
+// That direction is deliberate: over-counting says "changes pending" and
+// triggers a save that was not needed, while under-counting would tell a
+// backup script the dataset was clean when it was not.
+static KEYSPACE_CHANGE_COUNTERS: [PaddedCounter; COMMAND_COUNTER_SLOTS] =
+    [PADDED_COUNTER_ZERO; COMMAND_COUNTER_SLOTS];
+/// Value of the change counter when the last save completed. `rdb_changes_
+/// since_last_save` is the difference; a save that completes concurrently with
+/// writes can only make the difference smaller, never negative (saturating).
+static KEYSPACE_CHANGES_AT_LAST_SAVE: AtomicU64 = AtomicU64::new(0);
+
+/// Record one keyspace mutation. Hot path: one relaxed add, no allocation.
+#[inline]
+pub fn record_keyspace_change() {
+    COMMAND_COUNTER_SLOT.with(|&slot| {
+        KEYSPACE_CHANGE_COUNTERS[slot]
+            .0
+            .fetch_add(1, Ordering::Relaxed);
+    });
+}
+
+/// Exact sum across all slots. Read paths only (INFO).
+fn keyspace_changes_sum() -> u64 {
+    KEYSPACE_CHANGE_COUNTERS
+        .iter()
+        .map(|c| c.0.load(Ordering::Relaxed))
+        .sum()
+}
+
+/// Keyspace mutations since the last completed save (INFO `rdb_changes_since_last_save`).
+pub fn rdb_changes_since_last_save() -> u64 {
+    keyspace_changes_sum().saturating_sub(KEYSPACE_CHANGES_AT_LAST_SAVE.load(Ordering::Relaxed))
+}
+
+/// Mark a save as complete: subsequent changes count from here.
+///
+/// Called on SAVE / BGSAVE success, never on failure — a failed save left the
+/// dataset unpersisted, so the pending-change count must survive it.
+pub fn mark_save_completed() {
+    KEYSPACE_CHANGES_AT_LAST_SAVE.store(keyspace_changes_sum(), Ordering::Relaxed);
+}
+
+// ── EC9: replica sync counters (INFO `sync_full` / `sync_partial_*`) ─────
+// How an operator sees replicas thrashing: a climbing `sync_full` against a
+// flat `sync_partial_ok` means partial resync keeps failing and every replica
+// reconnect is re-shipping the whole dataset. Plain atomics — these fire once
+// per replica handshake, not per command.
+static SYNC_FULL: AtomicU64 = AtomicU64::new(0);
+static SYNC_PARTIAL_OK: AtomicU64 = AtomicU64::new(0);
+static SYNC_PARTIAL_ERR: AtomicU64 = AtomicU64::new(0);
+
+/// A replica was served a full resync (`+FULLRESYNC`).
+#[inline]
+pub fn record_sync_full() {
+    SYNC_FULL.fetch_add(1, Ordering::Relaxed);
+}
+
+/// A replica's `PSYNC <id> <offset>` was satisfied from the backlog.
+#[inline]
+pub fn record_sync_partial_ok() {
+    SYNC_PARTIAL_OK.fetch_add(1, Ordering::Relaxed);
+}
+
+/// A replica asked for a partial resync that could not be served.
+#[inline]
+pub fn record_sync_partial_err() {
+    SYNC_PARTIAL_ERR.fetch_add(1, Ordering::Relaxed);
+}
+
+/// Full resyncs served since start.
+pub fn sync_full() -> u64 {
+    SYNC_FULL.load(Ordering::Relaxed)
+}
+
+/// Partial resyncs served from the backlog since start.
+pub fn sync_partial_ok() -> u64 {
+    SYNC_PARTIAL_OK.load(Ordering::Relaxed)
+}
+
+/// Partial-resync requests that had to fall back to a full resync.
+pub fn sync_partial_err() -> u64 {
+    SYNC_PARTIAL_ERR.load(Ordering::Relaxed)
+}
+
 // ── P6: WAL aggressive reclamation counters (read by P10 INFO emitter) ───
 // Incremented by WalWriterV3::recycle_aggressive(). P10 reads these via the
 // public getters below to populate the `# Reclamation` INFO section.

src/command/connection.rs

@@ -204,6 +204,29 @@ fn run_id() -> &'static str {
     })
 }
 
+/// Process start instant, captured once at startup by [`record_server_start`].
+static SERVER_START: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();
+
+/// Capture the server start instant. Called from `main` before the listener
+/// binds, so `uptime_in_seconds` counts from a point that precedes the first
+/// client rather than from whenever INFO was first asked.
+pub fn record_server_start() {
+    let _ = SERVER_START.set(std::time::Instant::now());
+}
+
+/// Seconds since [`record_server_start`].
+///
+/// Falls back to initialising on first read, which keeps a unit test or an
+/// embedded harness that never calls `record_server_start` reporting a
+/// monotonic uptime instead of a constant — the only wrong answer here is one
+/// that never advances, because that is what makes a crash loop invisible.
+fn server_uptime_secs() -> u64 {
+    SERVER_START
+        .get_or_init(std::time::Instant::now)
+        .elapsed()
+        .as_secs()
+}
+
 /// Build the full INFO payload with every section present.
 ///
 /// Callers must pass this through [`crate::command::info_sections::finalize`],
@@ -238,6 +261,17 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
         sections.push_str("cluster_enabled:0\r\n");
     }
     let _ = write!(sections, "process_id:{}\r\n", std::process::id());
+    // The port this instance LISTENS on, not the port this connection arrived
+    // on. Behind a container port map or a proxy they differ, and a client
+    // handing a peer the arrival port would send it somewhere unreachable.
+    let _ = write!(sections, "tcp_port:{}\r\n", facts.tcp_port);
+    // Uptime is the field every restart-detector keys on: a drop to near zero
+    // is how a dashboard learns the process died. Sourced from the start
+    // instant captured before the listener binds, so it can never read as a
+    // constant.
+    let uptime_secs = server_uptime_secs();
+    let _ = write!(sections, "uptime_in_seconds:{uptime_secs}\r\n");
+    let _ = write!(sections, "uptime_in_days:{}\r\n", uptime_secs / 86_400);
     let _ = write!(sections, "os:{}\r\n", std::env::consts::OS);
     let _ = write!(
         sections,
@@ -401,9 +435,12 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
     };
     sections.push_str(&format!(
         "loading:0\r\n\
+         rdb_changes_since_last_save:{}\r\n\
          rdb_bgsave_in_progress:{}\r\n\
          rdb_last_save_time:{}\r\n\
          rdb_last_bgsave_status:{}\r\n\
+         aof_last_write_status:{}\r\n\
+         aof_last_bgrewrite_status:{}\r\n\
          aof_enabled:{}\r\n\
          aof_rewrite_in_progress:{}\r\n\
          aof_base_size:{}\r\n\
@@ -419,6 +456,10 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
          spill_failed_reinserted:{}\r\n\
          spill_completion_superseded:{}\r\n\
          spill_last_heartbeat_ms:{}\r\n",
+        // Keyspace mutations since the last COMPLETED save — the "is a save
+        // worth doing" signal a backup script reads. A failed save does not
+        // reset it: the dataset is still unpersisted.
+        crate::admin::metrics_setup::rdb_changes_since_last_save(),
         if crate::command::persistence::SAVE_IN_PROGRESS.load(std::sync::atomic::Ordering::Relaxed)
         {
             1
@@ -433,6 +474,20 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
         } else {
             "err"
         },
+        // Redis parity names for the two AOF statuses stock tooling
+        // string-matches. `aof_last_write_status` shares its source with
+        // Moon's own `aof_last_append_status` below — the same fact under the
+        // name a redis-py/ioredis health check actually looks for.
+        if crate::persistence::aof::AOF_LAST_APPEND_OK.load(std::sync::atomic::Ordering::Relaxed) {
+            "ok"
+        } else {
+            "err"
+        },
+        if crate::persistence::aof::AOF_REWRITE_LAST_OK.load(std::sync::atomic::Ordering::Relaxed) {
+            "ok"
+        } else {
+            "err"
+        },
         u8::from(aof_enabled),
         u8::from(
             crate::command::persistence::AOF_REWRITE_IN_PROGRESS
@@ -533,6 +588,9 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
          total_net_input_bytes:{}\r\n\
          total_net_output_bytes:{}\r\n\
          instantaneous_ops_per_sec:{}\r\n\
+         sync_full:{}\r\n\
+         sync_partial_ok:{}\r\n\
+         sync_partial_err:{}\r\n\
          pubsub_channels:{}\r\n\
          pubsub_patterns:{}\r\n",
         crate::admin::metrics_setup::keyspace_hits(),
@@ -543,6 +601,12 @@ fn info_raw(db: &Database, facts: &InstanceFacts) -> String {
         crate::admin::metrics_setup::total_net_input_bytes(),
         crate::admin::metrics_setup::total_net_output_bytes(),
         crate::admin::metrics_setup::instantaneous_ops_per_sec(),
+        // Replica-sync health: a climbing `sync_full` against a flat
+        // `sync_partial_ok` means partial resync keeps failing and every
+        // replica reconnect re-ships the whole dataset.
+        crate::admin::metrics_setup::sync_full(),
+        crate::admin::metrics_setup::sync_partial_ok(),
+        crate::admin::metrics_setup::sync_partial_err(),
         facts.pubsub_channels,
         facts.pubsub_patterns,
     );
@@ -655,6 +719,13 @@ pub struct InstanceFacts {
     pub pubsub_channels: usize,
     /// Distinct subscribed patterns, across all shards.
     pub pubsub_patterns: usize,
+    /// The port this instance's listener is bound to (`--port`).
+    ///
+    /// Deliberately the CONFIGURED port and not the local port of the socket
+    /// INFO arrived on: behind a container port map or a proxy the two differ,
+    /// and `tcp_port` exists so a client can hand a peer an address that
+    /// actually reaches this server.
+    pub tcp_port: u16,
 }
 
 /// As [`info_with_keyspace_and_replication`], plus the instance-wide facts