apollo_dashboard: alert on committer block number lag and block commit latency

crates/apollo_dashboard/resources/dev_grafana.json

@@ -1324,6 +1324,15 @@
           ],
           "extra_params": {}
         },
+        {
+          "title": "Committer Lag (blocks)",
+          "description": "Blocks the committer trails consensus (consensus_block_number - committer_offset), per namespace",
+          "type": "timeseries",
+          "exprs": [
+            "max by (namespace) (consensus_block_number{cluster=~\"$cluster\", namespace=~\"$namespace\"}) - max by (namespace) (committer_offset{cluster=~\"$cluster\", namespace=~\"$namespace\"})"
+          ],
+          "extra_params": {}
+        },
         {
           "title": "Total Block Duration",
           "description": "Average total block duration over a 1m window",
@@ -1336,6 +1345,19 @@
             "log_query": "\"Block\" AND \"durations in ms\""
           }
         },
+        {
+          "title": "Block Commit Latency",
+          "description": "Distribution of single-block commit duration",
+          "type": "timeseries",
+          "exprs": [
+            "histogram_quantile(0.50, sum by (le) (rate(committer_block_commit_latency_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))",
+            "histogram_quantile(0.95, sum by (le) (rate(committer_block_commit_latency_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))"
+          ],
+          "extra_params": {
+            "unit": "s",
+            "log_query": "\"Block\" AND \"durations in ms\""
+          }
+        },
         {
           "title": "Total Block Duration per Modification",
           "description": "Average total block duration per modification over a 1m window",

crates/apollo_dashboard/resources/dev_grafana_alerts.json

@@ -274,6 +274,58 @@
       "for": "30s",
       "severity": "p4"
     },
+    {
+      "name": "committer_block_commit_latency_too_high",
+      "title": "Committer block commit latency too high",
+      "ruleGroup": "evaluation_rate_default",
+      "expr": "((sum(rate(committer_block_commit_latency_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[2m])) or vector(0)) / clamp_min(sum(rate(committer_block_commit_latency_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[2m])) or vector(0), 0.0000001)) and on() (is_observer{cluster=~\"$cluster\", namespace=~\"$namespace\"} == 0)",
+      "conditions": [
+        {
+          "evaluator": {
+            "params": [
+              9.0
+            ],
+            "type": "gt"
+          },
+          "operator": {
+            "type": "and"
+          },
+          "reducer": {
+            "params": [],
+            "type": "avg"
+          },
+          "type": "query"
+        }
+      ],
+      "for": "30s",
+      "severity": "p4"
+    },
+    {
+      "name": "committer_block_number_lag",
+      "title": "Committer block number lag",
+      "ruleGroup": "evaluation_rate_default",
+      "expr": "(max by (namespace) (consensus_block_number{cluster=~\"$cluster\", namespace=~\"$namespace\"}) - max by (namespace) (committer_offset{cluster=~\"$cluster\", namespace=~\"$namespace\"})) and on (namespace) (is_observer{cluster=~\"$cluster\", namespace=~\"$namespace\"} == 0) and on (namespace) (increase(consensus_decisions_reached_by_sync{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) == 0)",
+      "conditions": [
+        {
+          "evaluator": {
+            "params": [
+              9.0
+            ],
+            "type": "gt"
+          },
+          "operator": {
+            "type": "and"
+          },
+          "reducer": {
+            "params": [],
+            "type": "avg"
+          },
+          "type": "query"
+        }
+      ],
+      "for": "5m",
+      "severity": "p2"
+    },
     {
       "name": "committer_remote_server_number_of_connections",
       "title": "committer - Remote server number of connections exceeds 80",

crates/apollo_dashboard/src/alert_definitions.rs

@@ -50,6 +50,8 @@ use crate::alert_scenarios::block_production_delay::{
     consensus_block_number_progress_is_slow,
     get_cende_write_blob_failure_alert,
     get_cende_write_blob_failure_once_alert,
+    get_committer_block_commit_latency_too_high,
+    get_committer_block_number_lag,
     get_consensus_p2p_peer_down,
     get_consensus_round_above_zero,
     get_consensus_round_above_zero_multiple_times,
@@ -649,6 +651,8 @@ pub fn get_apollo_alerts() -> Alerts {
     alerts.append(&mut get_batched_transactions_stuck_vec());
     alerts.push(consensus_block_number_progress_is_slow());
     alerts.push(get_cende_write_blob_failure_alert());
+    alerts.push(get_committer_block_commit_latency_too_high());
+    alerts.push(get_committer_block_number_lag());
     alerts.append(&mut get_consensus_block_number_stuck_vec());
     alerts.append(&mut get_consensus_p2p_not_enough_peers_for_quorum_vec());
     alerts.push(get_consensus_p2p_peer_down());

crates/apollo_dashboard/src/alert_scenarios/block_production_delay.rs

@@ -1,4 +1,10 @@
-use apollo_consensus::metrics::{CONSENSUS_BLOCK_NUMBER, CONSENSUS_ROUND_ABOVE_ZERO};
+use apollo_committer::metrics::{COMMITTER_BLOCK_COMMIT_LATENCY, COMMITTER_OFFSET};
+use apollo_consensus::metrics::{
+    CONSENSUS_BLOCK_NUMBER,
+    CONSENSUS_DECISIONS_REACHED_BY_SYNC,
+    CONSENSUS_ROUND_ABOVE_ZERO,
+    IS_OBSERVER,
+};
 use apollo_consensus_manager::metrics::CONSENSUS_NUM_CONNECTED_PEERS;
 use apollo_consensus_orchestrator::metrics::CENDE_WRITE_BLOB_FAILURE;
 use apollo_infra_utils::template::Template;
@@ -104,6 +110,62 @@ pub(crate) fn get_cende_write_blob_failure_once_alert() -> Alert {
     )
 }
 
+/// 9 fires at the STORED_BLOCK_HASH_BUFFER gate (10): integer gauges + strict `>`.
+const COMMITTER_LAG_ALERT_THRESHOLD_BLOCKS: f64 = 9.0;
+
+pub(crate) fn get_committer_block_number_lag() -> Alert {
+    Alert::new(
+        "committer_block_number_lag",
+        "Committer block number lag",
+        EvaluationRate::Default,
+        // Per-namespace lag (committer vs its own consensus); excludes observer and syncing nodes.
+        format!(
+            "(max by (namespace) ({consensus}) - max by (namespace) ({committer})) and on \
+             (namespace) ({observer} == 0) and on (namespace) (increase({sync}[5m]) == 0)",
+            consensus = CONSENSUS_BLOCK_NUMBER.get_name_with_filter(),
+            committer = COMMITTER_OFFSET.get_name_with_filter(),
+            observer = IS_OBSERVER.get_name_with_filter(),
+            sync = CONSENSUS_DECISIONS_REACHED_BY_SYNC.get_name_with_filter(),
+        ),
+        vec![AlertCondition::new(
+            AlertComparisonOp::GreaterThan,
+            COMMITTER_LAG_ALERT_THRESHOLD_BLOCKS,
+            AlertLogicalOp::And,
+        )],
+        // Debounce brief spikes before paging.
+        "5m",
+        AlertSeverity::Regular,
+        // Observers excluded in the expr; keep Applicable to avoid a second is_observer gate.
+        ObserverApplicability::Applicable,
+    )
+}
+
+/// Roughly 3x the committer's baseline commit latency.
+const COMMIT_LATENCY_ALERT_THRESHOLD_SECS: f64 = 9.0;
+
+pub(crate) fn get_committer_block_commit_latency_too_high() -> Alert {
+    Alert::new(
+        "committer_block_commit_latency_too_high",
+        "Committer block commit latency too high",
+        EvaluationRate::Default,
+        // 2m average commit latency, so a regression is caught before lag accumulates.
+        format!(
+            "(sum(rate({}[2m])) or vector(0)) / clamp_min(sum(rate({}[2m])) or vector(0), \
+             0.0000001)",
+            COMMITTER_BLOCK_COMMIT_LATENCY.get_name_sum_with_filter(),
+            COMMITTER_BLOCK_COMMIT_LATENCY.get_name_count_with_filter(),
+        ),
+        vec![AlertCondition::new(
+            AlertComparisonOp::GreaterThan,
+            COMMIT_LATENCY_ALERT_THRESHOLD_SECS,
+            AlertLogicalOp::And,
+        )],
+        PENDING_DURATION_DEFAULT,
+        AlertSeverity::WorkingHours,
+        ObserverApplicability::NotApplicable,
+    )
+}
+
 pub(crate) fn consensus_block_number_progress_is_slow() -> Alert {
     const ALERT_NAME: &str = "consensus_block_number_progress_is_slow";
     Alert::new(

crates/apollo_dashboard/src/panels/committer.rs

@@ -3,6 +3,7 @@ use apollo_committer::metrics::{
     AVERAGE_READ_RATE,
     AVERAGE_WRITE_RATE,
     BLOCKS_COMMITTED,
+    COMMITTER_BLOCK_COMMIT_LATENCY,
     COMMITTER_OFFSET,
     COMPUTE_DURATION_PER_BLOCK,
     COUNT_CLASSES_TRIE_MODIFICATIONS_PER_BLOCK,
@@ -14,6 +15,8 @@ use apollo_committer::metrics::{
     TOTAL_BLOCK_DURATION_PER_MODIFICATION,
     WRITE_DURATION_PER_BLOCK,
 };
+use apollo_consensus::metrics::CONSENSUS_BLOCK_NUMBER;
+use apollo_metrics::metric_definitions::POD_LABEL_FILTER;
 use apollo_metrics::metrics::MetricQueryName;
 
 use crate::dashboard::Row;
@@ -33,6 +36,19 @@ fn get_offset_panel() -> Panel {
     )
 }
 
+fn get_committer_lag_panel() -> Panel {
+    // Drop pod filter: the two metrics are on different pods, so a pod-scoped diff empties.
+    let consensus = CONSENSUS_BLOCK_NUMBER.get_name_with_filter().replace(POD_LABEL_FILTER, "");
+    let committer = COMMITTER_OFFSET.get_name_with_filter().replace(POD_LABEL_FILTER, "");
+    Panel::new(
+        "Committer Lag (blocks)",
+        "Blocks the committer trails consensus (consensus_block_number - committer_offset), per \
+         namespace",
+        format!("max by (namespace) ({consensus}) - max by (namespace) ({committer})"),
+        PanelType::TimeSeries,
+    )
+}
+
 /// Returns a panel that shows the average of a counter per block over a 1m window.
 fn average_per_block_panel(
     name: impl ToString,
@@ -70,6 +86,16 @@ fn get_total_block_duration_panel() -> Panel {
     )
 }
 
+fn get_block_commit_latency_panel() -> Panel {
+    Panel::from_hist(
+        &COMMITTER_BLOCK_COMMIT_LATENCY,
+        "Block Commit Latency",
+        "Distribution of single-block commit duration",
+    )
+    .with_unit(Unit::Seconds)
+    .with_log_query(BLOCK_DURATIONS_LOG_QUERY)
+}
+
 fn get_total_block_duration_per_modification_panel() -> Panel {
     average_per_block_panel(
         "Total Block Duration per Modification",
@@ -209,7 +235,9 @@ pub(crate) fn get_committer_row() -> Row {
         "Committer",
         vec![
             get_offset_panel(),
+            get_committer_lag_panel(),
             get_total_block_duration_panel(),
+            get_block_commit_latency_panel(),
             get_total_block_duration_per_modification_panel(),
             get_read_duration_per_block_panel(),
             get_average_read_rate_panel(),